[dpdk-dev] [PATCH 00/27] V1 event/dlb add Intel DLB PMD

[dpdk-dev] [PATCH 00/27] V1 event/dlb add Intel DLB PMD

[McDaniel, Timothy]

Hello Jerin and the DPDK community.

    The following patchset adds support for a new eventdev PMD. The DLB
    PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
    The DLB is a PCIe device that provides load-balanced, prioritized
    scheduling of core-to-core communication. The device consists of
    queues and arbiters that connect producer and consumer cores, and
    implements load-balanced queueing features including:
    - Lock-free multi-producer/multi-consumer operation.
    - Multiple priority levels for varying traffic types.
    - 'Direct' traffic (i.e. multi-producer/single-consumer)
    - Simple unordered load-balanced distribution.
    - Atomic lock-free load balancing across multiple consumers.
    - Queue element reordering feature allowing ordered load-balanced
      distribution.
   
    The DLB hardware supports both load balanced and directed ports and
    queues. Unlike other eventdev devices already in the repo,  not all
    DLB ports and queues are equally capable. In particular, directed
    ports are limited to a single link, and must be connected to a directed
    queue.
    Additionally, even though LDB ports may link multiple queues, the
    number of queues that may be linked is limited by hardware. Another
    difference is that DLB does not have a straightforward way of carrying
    the flow_id in the queue elements (QE) that the hardware operates on.
   
    Due to the above restrictions, we would like to extend the eventdev
    interface in manner that allows our PMD to take full advantage of
    the DLB hardware, while also adding useful functionality for non-Intel
    PMDs. Our proposed changes are contained in the first two patches
    in this patchset.
   
    While reviewing the code, please be aware that this PMD has full
    control over the DLB hardware. Intel will be extending the DLB PMD
    in the future (not as part of this first series) with a mode that we
    refer to as the bifurcated PMD. The bifurcated PMD communicates with a
    kernel driver to configure the device, ports, and queues, and memory
    maps device MMIO so datapath operations occur purely in user-space.
   
    The framework to support both the PF PMD and bifurcated PMD exists in
    this patchset, and is why the iface.[ch] layer is present.
   
    Thank You and Best Regards,
   
    Tim McDaniel


McDaniel, Timothy (27):
  eventdev: dlb upstream prerequisites
  eventdev: do not pass disable_implicit_release bit to trace macro
  event/dlb: add shared code version 10.7.9
  event/dlb: add make and meson build infrastructure
  event/dlb: add DLB documentation
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: inline functions used in multiple files
  event/dlb: add PFPMD-specific interface layer to shared code
  event/dlb: add flexible PMD to device interfaces
  event/dlb: add the PMD's public interfaces
  event/dlb: add xstats support
  event/dlb: add PMD self-tests
  event/dlb: add probe
  event/dlb: add infos_get and configure
  event/dlb: add queue_def_conf and port_def_conf
  event/dlb: add queue setup
  event/dlb: add port_setup
  event/dlb: add port_link
  event/dlb: add queue_release and port_release
  event/dlb: add port_unlink and port_unlinks_in_progress
  event/dlb: add eventdev_start
  event/dlb: add timout_ticks, dump, xstats, and selftest
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue, dequeue_burst, and variants
  event/dlb: add eventdev_stop and eventdev_close

 app/test-eventdev/evt_common.h                     |    1 +
 app/test-eventdev/test_order_atq.c                 |    4 +
 app/test-eventdev/test_order_common.c              |    6 +-
 app/test-eventdev/test_order_queue.c               |    4 +
 app/test-eventdev/test_perf_atq.c                  |    1 +
 app/test-eventdev/test_perf_queue.c                |    1 +
 app/test-eventdev/test_pipeline_atq.c              |    1 +
 app/test-eventdev/test_pipeline_queue.c            |    1 +
 app/test/test_eventdev.c                           |    4 +-
 config/common_base                                 |   17 +
 config/rte_config.h                                |    6 +
 doc/guides/eventdevs/dlb.rst                       |  497 +
 drivers/event/Makefile                             |    5 +
 drivers/event/dlb/Makefile                         |   36 +
 drivers/event/dlb/dlb.c                            | 4306 +++++++++
 drivers/event/dlb/dlb_iface.c                      |  105 +
 drivers/event/dlb/dlb_iface.h                      |   92 +
 drivers/event/dlb/dlb_inline_fns.h                 |   80 +
 drivers/event/dlb/dlb_log.h                        |   24 +
 drivers/event/dlb/dlb_priv.h                       |  595 ++
 drivers/event/dlb/dlb_selftest.c                   | 1628 ++++
 drivers/event/dlb/dlb_user.h                       | 1351 +++
 drivers/event/dlb/dlb_xstats.c                     | 1251 +++
 drivers/event/dlb/meson.build                      |   16 +
 drivers/event/dlb/pf/base/dlb_hw_types.h           |  360 +
 drivers/event/dlb/pf/base/dlb_mbox.h               |  645 ++
 drivers/event/dlb/pf/base/dlb_osdep.h              |  350 +
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h       |  449 +
 drivers/event/dlb/pf/base/dlb_osdep_list.h         |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h        |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h               | 2646 ++++++
 drivers/event/dlb/pf/base/dlb_resource.c           | 9699 ++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h           | 1625 ++++
 drivers/event/dlb/pf/base/dlb_user.h               | 1084 +++
 drivers/event/dlb/pf/dlb_main.c                    |  670 ++
 drivers/event/dlb/pf/dlb_main.h                    |   90 +
 drivers/event/dlb/pf/dlb_pf.c                      |  839 ++
 drivers/event/dlb/rte_pmd_dlb.c                    |   39 +
 drivers/event/dlb/rte_pmd_dlb.h                    |   69 +
 drivers/event/dlb/rte_pmd_dlb_event_version.map    |    6 +
 drivers/event/dpaa2/dpaa2_eventdev.c               |    2 +-
 drivers/event/meson.build                          |    3 +
 drivers/event/octeontx/ssovf_evdev.c               |    2 +-
 drivers/event/skeleton/skeleton_eventdev.c         |    2 +-
 drivers/event/sw/sw_evdev.c                        |    5 +-
 drivers/event/sw/sw_evdev_selftest.c               |    9 +-
 .../eventdev_pipeline/pipeline_worker_generic.c    |    8 +-
 examples/eventdev_pipeline/pipeline_worker_tx.c    |    3 +
 examples/l2fwd-event/l2fwd_event_generic.c         |    5 +-
 examples/l2fwd-event/l2fwd_event_internal_port.c   |    5 +-
 examples/l3fwd/l3fwd_event_generic.c               |    5 +-
 examples/l3fwd/l3fwd_event_internal_port.c         |    5 +-
 lib/librte_eal/x86/include/rte_cpuflags.h          |    1 +
 lib/librte_eal/x86/rte_cpuflags.c                  |    1 +
 lib/librte_eventdev/rte_event_crypto_adapter.c     |    7 +-
 lib/librte_eventdev/rte_event_eth_tx_adapter.c     |   16 +-
 lib/librte_eventdev/rte_eventdev.c                 |   72 +-
 lib/librte_eventdev/rte_eventdev.h                 |   51 +-
 lib/librte_eventdev/rte_eventdev_pmd_pci.h         |   54 +
 lib/librte_eventdev/rte_eventdev_trace.h           |   15 +-
 60 files changed, 28987 insertions(+), 49 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/Makefile
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_mbox.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_user.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/rte_pmd_dlb_event_version.map

-- 
2.13.6

[dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

[McDaniel, Timothy]

The DLB hardware does not conform exactly to the eventdev interface.
1) It has a limit on the number of queues that may be linked to a port.
2) Some ports a further restricted to a maximum of 1 linked queue.
3) It does not (currently) have the ability to carry the flow_id as part
of the event (QE) payload.

Due to the above, we would like to propose the following enhancements.

1) Add new fields to the rte_event_dev_info struct. These fields allow
the device to advertize its capabilities so that applications can take
the appropriate actions based on those capabilities.

    struct rte_event_dev_info {
	uint32_t max_event_port_links;
	/**< Maximum number of queues that can be linked to a single event
	 * port by this device.
	 */

	uint8_t max_single_link_event_port_queue_pairs;
	/**< Maximum number of event ports and queues that are optimized for
	 * (and only capable of) single-link configurations supported by this
	 * device. These ports and queues are not accounted for in
	 * max_event_ports or max_event_queues.
	 */
    }

2) Add a new field to the rte_event_dev_config struct. This field allows the
application to specify how many of its ports are limited to a single link,
or will be used in single link mode.

    /** Event device configuration structure */
    struct rte_event_dev_config {
	uint8_t nb_single_link_event_port_queues;
	/**< Number of event ports and queues that will be singly-linked to
	 * each other. These are a subset of the overall event ports and
	 * queues; this value cannot exceed *nb_event_ports* or
	 * *nb_event_queues*. If the device has ports and queues that are
	 * optimized for single-link usage, this field is a hint for how many
	 * to allocate; otherwise, regular event ports and queues can be used.
	 */
    }

3) Replace the dedicated implicit_release_disabled field with a bit field
of explicit port capabilities. The implicit_release_disable functionality
is assiged to one bit, and a port-is-single-link-only  attribute is
assigned to other, with the remaining bits available for future assignment.

	* Event port configuration bitmap flags */
	#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
	/**< Configure the port not to release outstanding events in
	 * rte_event_dev_dequeue_burst(). If set, all events received through
	 * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
	 * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
	 * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
	 */
	#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)

	/**< This event port links only to a single event queue.
	 *
	 *  @see rte_event_port_setup(), rte_event_port_link()
	 */

	#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
	/**
	 * The implicit release disable attribute of the port
	 */

	struct rte_event_port_conf {
		uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
	}

4) Add UMWAIT/UMONITOR bit to rte_cpuflags

5) Added a new API that is useful for probing PCI devices.

	/**
	 * @internal
	 * Wrapper for use by pci drivers as a .probe function to attach to a event
	 * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
	 * the name.
	 */
	static inline int
	rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
				    struct rte_pci_device *pci_dev,
				    size_t private_data_size,
				    eventdev_pmd_pci_callback_t devinit,
				    const char *name);

Change-Id: I4cf00015296e2b3feca9886895765554730594be
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 app/test-eventdev/evt_common.h                     |  1 +
 app/test-eventdev/test_order_atq.c                 |  4 ++
 app/test-eventdev/test_order_common.c              |  6 ++-
 app/test-eventdev/test_order_queue.c               |  4 ++
 app/test-eventdev/test_perf_atq.c                  |  1 +
 app/test-eventdev/test_perf_queue.c                |  1 +
 app/test-eventdev/test_pipeline_atq.c              |  1 +
 app/test-eventdev/test_pipeline_queue.c            |  1 +
 app/test/test_eventdev.c                           |  4 +-
 drivers/event/dpaa2/dpaa2_eventdev.c               |  2 +-
 drivers/event/octeontx/ssovf_evdev.c               |  2 +-
 drivers/event/skeleton/skeleton_eventdev.c         |  2 +-
 drivers/event/sw/sw_evdev.c                        |  5 +-
 drivers/event/sw/sw_evdev_selftest.c               |  9 ++--
 .../eventdev_pipeline/pipeline_worker_generic.c    |  8 ++-
 examples/eventdev_pipeline/pipeline_worker_tx.c    |  3 ++
 examples/l2fwd-event/l2fwd_event_generic.c         |  5 +-
 examples/l2fwd-event/l2fwd_event_internal_port.c   |  5 +-
 examples/l3fwd/l3fwd_event_generic.c               |  5 +-
 examples/l3fwd/l3fwd_event_internal_port.c         |  5 +-
 lib/librte_eal/x86/include/rte_cpuflags.h          |  1 +
 lib/librte_eal/x86/rte_cpuflags.c                  |  1 +
 lib/librte_eventdev/rte_event_eth_tx_adapter.c     |  2 +-
 lib/librte_eventdev/rte_eventdev.c                 | 62 +++++++++++++++++++---
 lib/librte_eventdev/rte_eventdev.h                 | 51 +++++++++++++++---
 lib/librte_eventdev/rte_eventdev_pmd_pci.h         | 54 +++++++++++++++++++
 26 files changed, 208 insertions(+), 37 deletions(-)

diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
index f9d7378d3..120c27b33 100644
--- a/app/test-eventdev/evt_common.h
+++ b/app/test-eventdev/evt_common.h
@@ -169,6 +169,7 @@ evt_configure_eventdev(struct evt_options *opt, uint8_t nb_queues,
 			.dequeue_timeout_ns = opt->deq_tmo_nsec,
 			.nb_event_queues = nb_queues,
 			.nb_event_ports = nb_ports,
+			.nb_single_link_event_port_queues = 0,
 			.nb_events_limit  = info.max_num_events,
 			.nb_event_queue_flows = opt->nb_flows,
 			.nb_event_port_dequeue_depth =
diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
index 3366cfce9..8246b96f0 100644
--- a/app/test-eventdev/test_order_atq.c
+++ b/app/test-eventdev/test_order_atq.c
@@ -34,6 +34,8 @@ order_atq_worker(void *arg)
 			continue;
 		}
 
+		ev.flow_id = ev.mbuf->udata64;
+
 		if (ev.sub_event_type == 0) { /* stage 0 from producer */
 			order_atq_process_stage_0(&ev);
 			while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
@@ -68,6 +70,8 @@ order_atq_worker_burst(void *arg)
 		}
 
 		for (i = 0; i < nb_rx; i++) {
+			ev[i].flow_id = ev[i].mbuf->udata64;
+
 			if (ev[i].sub_event_type == 0) { /*stage 0 */
 				order_atq_process_stage_0(&ev[i]);
 			} else if (ev[i].sub_event_type == 1) { /* stage 1 */
diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
index 4190f9ade..c6fcd0509 100644
--- a/app/test-eventdev/test_order_common.c
+++ b/app/test-eventdev/test_order_common.c
@@ -49,6 +49,7 @@ order_producer(void *arg)
 		const uint32_t flow = (uintptr_t)m % nb_flows;
 		/* Maintain seq number per flow */
 		m->seqn = producer_flow_seq[flow]++;
+		m->udata64 = flow;
 
 		ev.flow_id = flow;
 		ev.mbuf = m;
@@ -318,10 +319,11 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
 		opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
 
 	/* port configuration */
-	const struct rte_event_port_conf p_conf = {
+	struct rte_event_port_conf p_conf = {
 			.dequeue_depth = opt->wkr_deq_dep,
 			.enqueue_depth = dev_info.max_event_port_dequeue_depth,
 			.new_event_threshold = dev_info.max_num_events,
+			.event_port_cfg = 0,
 	};
 
 	/* setup one port per worker, linking to all queues */
@@ -351,6 +353,8 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
 	p->queue_id = 0;
 	p->t = t;
 
+	p_conf.new_event_threshold /= 2;
+
 	ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
 	if (ret) {
 		evt_err("failed to setup producer port %d", port);
diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
index 495efd92f..a0a2187a2 100644
--- a/app/test-eventdev/test_order_queue.c
+++ b/app/test-eventdev/test_order_queue.c
@@ -34,6 +34,8 @@ order_queue_worker(void *arg)
 			continue;
 		}
 
+		ev.flow_id = ev.mbuf->udata64;
+
 		if (ev.queue_id == 0) { /* from ordered queue */
 			order_queue_process_stage_0(&ev);
 			while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
@@ -68,6 +70,8 @@ order_queue_worker_burst(void *arg)
 		}
 
 		for (i = 0; i < nb_rx; i++) {
+			ev[i].flow_id = ev[i].mbuf->udata64;
+
 			if (ev[i].queue_id == 0) { /* from ordered queue */
 				order_queue_process_stage_0(&ev[i]);
 			} else if (ev[i].queue_id == 1) {/* from atomic queue */
diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
index 8fd51004e..10846f202 100644
--- a/app/test-eventdev/test_perf_atq.c
+++ b/app/test-eventdev/test_perf_atq.c
@@ -204,6 +204,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 			.dequeue_depth = opt->wkr_deq_dep,
 			.enqueue_depth = dev_info.max_event_port_dequeue_depth,
 			.new_event_threshold = dev_info.max_num_events,
+			.event_port_cfg = 0,
 	};
 
 	ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues,
diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
index f4ea3a795..a0119da60 100644
--- a/app/test-eventdev/test_perf_queue.c
+++ b/app/test-eventdev/test_perf_queue.c
@@ -219,6 +219,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 			.dequeue_depth = opt->wkr_deq_dep,
 			.enqueue_depth = dev_info.max_event_port_dequeue_depth,
 			.new_event_threshold = dev_info.max_num_events,
+			.event_port_cfg = 0,
 	};
 
 	ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */,
diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c
index 8e8686c14..a95ec0aa5 100644
--- a/app/test-eventdev/test_pipeline_atq.c
+++ b/app/test-eventdev/test_pipeline_atq.c
@@ -356,6 +356,7 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 		.dequeue_depth = opt->wkr_deq_dep,
 		.enqueue_depth = info.max_event_port_dequeue_depth,
 		.new_event_threshold = info.max_num_events,
+		.event_port_cfg = 0,
 	};
 
 	if (!t->internal_port)
diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c
index 7bebac34f..30817dc78 100644
--- a/app/test-eventdev/test_pipeline_queue.c
+++ b/app/test-eventdev/test_pipeline_queue.c
@@ -379,6 +379,7 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 			.dequeue_depth = opt->wkr_deq_dep,
 			.enqueue_depth = info.max_event_port_dequeue_depth,
 			.new_event_threshold = info.max_num_events,
+			.event_port_cfg = 0,
 	};
 
 	if (!t->internal_port) {
diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 43ccb1ce9..62019c185 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -559,10 +559,10 @@ test_eventdev_port_setup(void)
 	if (!(info.event_dev_cap &
 	      RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
 		pconf.enqueue_depth = info.max_event_port_enqueue_depth;
-		pconf.disable_implicit_release = 1;
+		pconf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
 		ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf);
 		TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret);
-		pconf.disable_implicit_release = 0;
+		pconf.event_port_cfg = 0;
 	}
 
 	ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports,
diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
index a196ad4c6..8568bfcfc 100644
--- a/drivers/event/dpaa2/dpaa2_eventdev.c
+++ b/drivers/event/dpaa2/dpaa2_eventdev.c
@@ -537,7 +537,7 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 		DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
 	port_conf->enqueue_depth =
 		DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
-	port_conf->disable_implicit_release = 0;
+	port_conf->event_port_cfg = 0;
 }
 
 static int
diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
index 1b1a5d939..99c0b2efb 100644
--- a/drivers/event/octeontx/ssovf_evdev.c
+++ b/drivers/event/octeontx/ssovf_evdev.c
@@ -224,7 +224,7 @@ ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 	port_conf->new_event_threshold = edev->max_num_events;
 	port_conf->dequeue_depth = 1;
 	port_conf->enqueue_depth = 1;
-	port_conf->disable_implicit_release = 0;
+	port_conf->event_port_cfg = 0;
 }
 
 static void
diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
index c889220e0..37d569b8c 100644
--- a/drivers/event/skeleton/skeleton_eventdev.c
+++ b/drivers/event/skeleton/skeleton_eventdev.c
@@ -209,7 +209,7 @@ skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 	port_conf->new_event_threshold = 32 * 1024;
 	port_conf->dequeue_depth = 16;
 	port_conf->enqueue_depth = 16;
-	port_conf->disable_implicit_release = 0;
+	port_conf->event_port_cfg = 0;
 }
 
 static void
diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index fb8e8bebb..0b3dd9c1c 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -175,7 +175,8 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 	}
 
 	p->inflight_max = conf->new_event_threshold;
-	p->implicit_release = !conf->disable_implicit_release;
+	p->implicit_release = !(conf->event_port_cfg &
+				RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
 
 	/* check if ring exists, same as rx_worker above */
 	snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
@@ -508,7 +509,7 @@ sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 	port_conf->new_event_threshold = 1024;
 	port_conf->dequeue_depth = 16;
 	port_conf->enqueue_depth = 16;
-	port_conf->disable_implicit_release = 0;
+	port_conf->event_port_cfg = 0;
 }
 
 static int
diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
index 38c21fa0f..a78d6cd0d 100644
--- a/drivers/event/sw/sw_evdev_selftest.c
+++ b/drivers/event/sw/sw_evdev_selftest.c
@@ -172,7 +172,7 @@ create_ports(struct test *t, int num_ports)
 			.new_event_threshold = 1024,
 			.dequeue_depth = 32,
 			.enqueue_depth = 64,
-			.disable_implicit_release = 0,
+			.event_port_cfg = 0,
 	};
 	if (num_ports > MAX_PORTS)
 		return -1;
@@ -1227,7 +1227,7 @@ port_reconfig_credits(struct test *t)
 				.new_event_threshold = 128,
 				.dequeue_depth = 32,
 				.enqueue_depth = 64,
-				.disable_implicit_release = 0,
+				.event_port_cfg = 0,
 		};
 		if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
 			printf("%d Error setting up port\n", __LINE__);
@@ -1317,7 +1317,7 @@ port_single_lb_reconfig(struct test *t)
 		.new_event_threshold = 128,
 		.dequeue_depth = 32,
 		.enqueue_depth = 64,
-		.disable_implicit_release = 0,
+		.event_port_cfg = 0,
 	};
 	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
 		printf("%d Error setting up port\n", __LINE__);
@@ -3079,7 +3079,8 @@ worker_loopback(struct test *t, uint8_t disable_implicit_release)
 	 * only be initialized once - and this needs to be set for multiple runs
 	 */
 	conf.new_event_threshold = 512;
-	conf.disable_implicit_release = disable_implicit_release;
+	conf.event_port_cfg = disable_implicit_release ?
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
 
 	if (rte_event_port_setup(evdev, 0, &conf) < 0) {
 		printf("Error setting up RX port\n");
diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
index 42ff4eeb9..a091da3ba 100644
--- a/examples/eventdev_pipeline/pipeline_worker_generic.c
+++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
@@ -129,6 +129,7 @@ setup_eventdev_generic(struct worker_data *worker_data)
 	struct rte_event_dev_config config = {
 			.nb_event_queues = nb_queues,
 			.nb_event_ports = nb_ports,
+			.nb_single_link_event_port_queues = 1,
 			.nb_events_limit  = 4096,
 			.nb_event_queue_flows = 1024,
 			.nb_event_port_dequeue_depth = 128,
@@ -138,12 +139,13 @@ setup_eventdev_generic(struct worker_data *worker_data)
 			.dequeue_depth = cdata.worker_cq_depth,
 			.enqueue_depth = 64,
 			.new_event_threshold = 4096,
+			.event_port_cfg = 0,
 	};
 	struct rte_event_queue_conf wkr_q_conf = {
 			.schedule_type = cdata.queue_type,
 			.priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 			.nb_atomic_flows = 1024,
-		.nb_atomic_order_sequences = 1024,
+			.nb_atomic_order_sequences = 1024,
 	};
 	struct rte_event_queue_conf tx_q_conf = {
 			.priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
@@ -167,7 +169,8 @@ setup_eventdev_generic(struct worker_data *worker_data)
 	disable_implicit_release = (dev_info.event_dev_cap &
 			RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
 
-	wkr_p_conf.disable_implicit_release = disable_implicit_release;
+	wkr_p_conf.event_port_cfg = disable_implicit_release ?
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
 
 	if (dev_info.max_num_events < config.nb_events_limit)
 		config.nb_events_limit = dev_info.max_num_events;
@@ -417,6 +420,7 @@ init_adapters(uint16_t nb_ports)
 		.dequeue_depth = cdata.worker_cq_depth,
 		.enqueue_depth = 64,
 		.new_event_threshold = 4096,
+		.event_port_cfg = 0,
 	};
 
 	if (adptr_p_conf.new_event_threshold > dev_info.max_num_events)
diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
index 55bb2f762..e8a9652aa 100644
--- a/examples/eventdev_pipeline/pipeline_worker_tx.c
+++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
@@ -436,6 +436,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
 	struct rte_event_dev_config config = {
 			.nb_event_queues = nb_queues,
 			.nb_event_ports = nb_ports,
+			.nb_single_link_event_port_queues = 0,
 			.nb_events_limit  = 4096,
 			.nb_event_queue_flows = 1024,
 			.nb_event_port_dequeue_depth = 128,
@@ -445,6 +446,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
 			.dequeue_depth = cdata.worker_cq_depth,
 			.enqueue_depth = 64,
 			.new_event_threshold = 4096,
+			.event_port_cfg = 0,
 	};
 	struct rte_event_queue_conf wkr_q_conf = {
 			.schedule_type = cdata.queue_type,
@@ -746,6 +748,7 @@ init_adapters(uint16_t nb_ports)
 		.dequeue_depth = cdata.worker_cq_depth,
 		.enqueue_depth = 64,
 		.new_event_threshold = 4096,
+		.event_port_cfg = 0,
 	};
 
 	init_ports(nb_ports);
diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
index 2dc95e5f7..e01df0435 100644
--- a/examples/l2fwd-event/l2fwd_event_generic.c
+++ b/examples/l2fwd-event/l2fwd_event_generic.c
@@ -126,8 +126,9 @@ l2fwd_event_port_setup_generic(struct l2fwd_resources *rsrc)
 	if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
 		event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
 
-	event_p_conf.disable_implicit_release =
-		evt_rsrc->disable_implicit_release;
+	event_p_conf.event_port_cfg = 0;
+	if (evt_rsrc->disable_implicit_release)
+		event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
 	evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
 
 	for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
index 63d57b46c..f54327b4f 100644
--- a/examples/l2fwd-event/l2fwd_event_internal_port.c
+++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
@@ -123,8 +123,9 @@ l2fwd_event_port_setup_internal_port(struct l2fwd_resources *rsrc)
 	if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
 		event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
 
-	event_p_conf.disable_implicit_release =
-		evt_rsrc->disable_implicit_release;
+	event_p_conf.event_port_cfg = 0;
+	if (evt_rsrc->disable_implicit_release)
+		event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
 
 	for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
 								event_p_id++) {
diff --git a/examples/l3fwd/l3fwd_event_generic.c b/examples/l3fwd/l3fwd_event_generic.c
index f8c98435d..409a4107e 100644
--- a/examples/l3fwd/l3fwd_event_generic.c
+++ b/examples/l3fwd/l3fwd_event_generic.c
@@ -115,8 +115,9 @@ l3fwd_event_port_setup_generic(void)
 	if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
 		event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
 
-	event_p_conf.disable_implicit_release =
-		evt_rsrc->disable_implicit_release;
+	event_p_conf.event_port_cfg = 0;
+	if (evt_rsrc->disable_implicit_release)
+		event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
 	evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
 
 	for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
diff --git a/examples/l3fwd/l3fwd_event_internal_port.c b/examples/l3fwd/l3fwd_event_internal_port.c
index 03ac581d6..df410f10f 100644
--- a/examples/l3fwd/l3fwd_event_internal_port.c
+++ b/examples/l3fwd/l3fwd_event_internal_port.c
@@ -113,8 +113,9 @@ l3fwd_event_port_setup_internal_port(void)
 	if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
 		event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
 
-	event_p_conf.disable_implicit_release =
-		evt_rsrc->disable_implicit_release;
+	event_p_conf.event_port_cfg = 0;
+	if (evt_rsrc->disable_implicit_release)
+		event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
 
 	for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
 								event_p_id++) {
diff --git a/lib/librte_eal/x86/include/rte_cpuflags.h b/lib/librte_eal/x86/include/rte_cpuflags.h
index c1d20364d..ab2c3b379 100644
--- a/lib/librte_eal/x86/include/rte_cpuflags.h
+++ b/lib/librte_eal/x86/include/rte_cpuflags.h
@@ -130,6 +130,7 @@ enum rte_cpu_flag_t {
 	RTE_CPUFLAG_CLDEMOTE,               /**< Cache Line Demote */
 	RTE_CPUFLAG_MOVDIRI,                /**< Direct Store Instructions */
 	RTE_CPUFLAG_MOVDIR64B,              /**< Direct Store Instructions 64B */
+	RTE_CPUFLAG_UMWAIT,                 /**< UMONITOR/UMWAIT */
 	RTE_CPUFLAG_AVX512VP2INTERSECT,     /**< AVX512 Two Register Intersection */
 
 	/* The last item */
diff --git a/lib/librte_eal/x86/rte_cpuflags.c b/lib/librte_eal/x86/rte_cpuflags.c
index 30439e795..69ac0dbce 100644
--- a/lib/librte_eal/x86/rte_cpuflags.c
+++ b/lib/librte_eal/x86/rte_cpuflags.c
@@ -137,6 +137,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
 	FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25)
 	FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27)
 	FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28)
+        FEAT_DEF(UMWAIT, 0x00000007, 0, RTE_REG_ECX, 5)
 	FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8)
 };
 
diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
index bb21dc407..8a72256de 100644
--- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
+++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
@@ -286,7 +286,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id,
 		return ret;
 	}
 
-	pc->disable_implicit_release = 0;
+	pc->event_port_cfg = 0;
 	ret = rte_event_port_setup(dev_id, port_id, pc);
 	if (ret) {
 		RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
index 82c177c73..4955ab1a0 100644
--- a/lib/librte_eventdev/rte_eventdev.c
+++ b/lib/librte_eventdev/rte_eventdev.c
@@ -437,9 +437,29 @@ rte_event_dev_configure(uint8_t dev_id,
 					dev_id);
 		return -EINVAL;
 	}
-	if (dev_conf->nb_event_queues > info.max_event_queues) {
-		RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d",
-		dev_id, dev_conf->nb_event_queues, info.max_event_queues);
+	if (dev_conf->nb_event_queues > info.max_event_queues +
+			info.max_single_link_event_port_queue_pairs) {
+		RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d + max_single_link_event_port_queue_pairs=%d",
+				 dev_id, dev_conf->nb_event_queues,
+				 info.max_event_queues,
+				 info.max_single_link_event_port_queue_pairs);
+		return -EINVAL;
+	}
+	if (dev_conf->nb_event_queues -
+			dev_conf->nb_single_link_event_port_queues >
+			info.max_event_queues) {
+		RTE_EDEV_LOG_ERR("id%d nb_event_queues=%d - nb_single_link_event_port_queues=%d > max_event_queues=%d",
+				 dev_id, dev_conf->nb_event_queues,
+				 dev_conf->nb_single_link_event_port_queues,
+				 info.max_event_queues);
+		return -EINVAL;
+	}
+	if (dev_conf->nb_single_link_event_port_queues >
+			dev_conf->nb_event_queues) {
+		RTE_EDEV_LOG_ERR("dev%d nb_single_link_event_port_queues=%d > nb_event_queues=%d",
+				 dev_id,
+				 dev_conf->nb_single_link_event_port_queues,
+				 dev_conf->nb_event_queues);
 		return -EINVAL;
 	}
 
@@ -448,9 +468,31 @@ rte_event_dev_configure(uint8_t dev_id,
 		RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id);
 		return -EINVAL;
 	}
-	if (dev_conf->nb_event_ports > info.max_event_ports) {
-		RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d",
-		dev_id, dev_conf->nb_event_ports, info.max_event_ports);
+	if (dev_conf->nb_event_ports > info.max_event_ports +
+			info.max_single_link_event_port_queue_pairs) {
+		RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports=%d + max_single_link_event_port_queue_pairs=%d",
+				 dev_id, dev_conf->nb_event_ports,
+				 info.max_event_ports,
+				 info.max_single_link_event_port_queue_pairs);
+		return -EINVAL;
+	}
+	if (dev_conf->nb_event_ports -
+			dev_conf->nb_single_link_event_port_queues
+			> info.max_event_ports) {
+		RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d - nb_single_link_event_port_queues=%d > max_event_ports=%d",
+				 dev_id, dev_conf->nb_event_ports,
+				 dev_conf->nb_single_link_event_port_queues,
+				 info.max_event_ports);
+		return -EINVAL;
+	}
+
+	if (dev_conf->nb_single_link_event_port_queues >
+	    dev_conf->nb_event_ports) {
+		RTE_EDEV_LOG_ERR(
+				 "dev%d nb_single_link_event_port_queues=%d > nb_event_ports=%d",
+				 dev_id,
+				 dev_conf->nb_single_link_event_port_queues,
+				 dev_conf->nb_event_ports);
 		return -EINVAL;
 	}
 
@@ -737,7 +779,8 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
 		return -EINVAL;
 	}
 
-	if (port_conf && port_conf->disable_implicit_release &&
+	if (port_conf &&
+	    (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL) &&
 	    !(dev->data->event_dev_cap &
 	      RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
 		RTE_EDEV_LOG_ERR(
@@ -809,6 +852,7 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
 			uint32_t *attr_value)
 {
 	struct rte_eventdev *dev;
+	uint32_t config;
 
 	if (!attr_value)
 		return -EINVAL;
@@ -830,6 +874,10 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
 	case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD:
 		*attr_value = dev->data->ports_cfg[port_id].new_event_threshold;
 		break;
+	case RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE:
+		config = dev->data->ports_cfg[port_id].event_port_cfg;
+		*attr_value = !!(config & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+		break;
 	default:
 		return -EINVAL;
 	};
diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
index 7dc832353..7f7a8a275 100644
--- a/lib/librte_eventdev/rte_eventdev.h
+++ b/lib/librte_eventdev/rte_eventdev.h
@@ -291,6 +291,13 @@ struct rte_event;
  * single queue to each port or map a single queue to many port.
  */
 
+#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)
+/**< Event device is capable of carrying the flow ID from the enqueued
+ * event to the dequeued event. If the flag is set, the dequeued event's flow
+ * ID matches the corresponding enqueued event's flow ID. If the flag is not
+ * set, the dequeued event's flow ID field is uninitialized.
+ */
+
 /* Event device priority levels */
 #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
 /**< Highest priority expressed across eventdev subsystem
@@ -380,6 +387,10 @@ struct rte_event_dev_info {
 	 * event port by this device.
 	 * A device that does not support bulk enqueue will set this as 1.
 	 */
+	uint32_t max_event_port_links;
+	/**< Maximum number of queues that can be linked to a single event
+	 * port by this device.
+	 */
 	int32_t max_num_events;
 	/**< A *closed system* event dev has a limit on the number of events it
 	 * can manage at a time. An *open system* event dev does not have a
@@ -387,6 +398,12 @@ struct rte_event_dev_info {
 	 */
 	uint32_t event_dev_cap;
 	/**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/
+	uint8_t max_single_link_event_port_queue_pairs;
+	/**< Maximum number of event ports and queues that are optimized for
+	 * (and only capable of) single-link configurations supported by this
+	 * device. These ports and queues are not accounted for in
+	 * max_event_ports or max_event_queues.
+	 */
 };
 
 /**
@@ -494,6 +511,14 @@ struct rte_event_dev_config {
 	 */
 	uint32_t event_dev_cfg;
 	/**< Event device config flags(RTE_EVENT_DEV_CFG_)*/
+	uint8_t nb_single_link_event_port_queues;
+	/**< Number of event ports and queues that will be singly-linked to
+	 * each other. These are a subset of the overall event ports and
+	 * queues; this value cannot exceed *nb_event_ports* or
+	 * *nb_event_queues*. If the device has ports and queues that are
+	 * optimized for single-link usage, this field is a hint for how many
+	 * to allocate; otherwise, regular event ports and queues can be used.
+	 */
 };
 
 /**
@@ -671,6 +696,20 @@ rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
 
 /* Event port specific APIs */
 
+/* Event port configuration bitmap flags */
+#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
+/**< Configure the port not to release outstanding events in
+ * rte_event_dev_dequeue_burst(). If set, all events received through
+ * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
+ * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
+ * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
+ */
+#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
+/**< This event port links only to a single event queue.
+ *
+ *  @see rte_event_port_setup(), rte_event_port_link()
+ */
+
 /** Event port configuration structure */
 struct rte_event_port_conf {
 	int32_t new_event_threshold;
@@ -698,13 +737,7 @@ struct rte_event_port_conf {
 	 * which previously supplied to rte_event_dev_configure().
 	 * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable.
 	 */
-	uint8_t disable_implicit_release;
-	/**< Configure the port not to release outstanding events in
-	 * rte_event_dev_dequeue_burst(). If true, all events received through
-	 * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
-	 * RTE_EVENT_OP_FORWARD. Must be false when the device is not
-	 * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
-	 */
+	uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
 };
 
 /**
@@ -769,6 +802,10 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
  * The new event threshold of the port
  */
 #define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2
+/**
+ * The implicit release disable attribute of the port
+ */
+#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
 
 /**
  * Get an attribute from a port.
diff --git a/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
index 443cd38c2..157299983 100644
--- a/lib/librte_eventdev/rte_eventdev_pmd_pci.h
+++ b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
@@ -88,6 +88,60 @@ rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv,
 	return -ENXIO;
 }
 
+/**
+ * @internal
+ * Wrapper for use by pci drivers as a .probe function to attach to a event
+ * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
+ * the name.
+ */
+static inline int
+rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
+			    struct rte_pci_device *pci_dev,
+			    size_t private_data_size,
+			    eventdev_pmd_pci_callback_t devinit,
+			    const char *name)
+{
+	struct rte_eventdev *eventdev;
+
+	int retval;
+
+	if (devinit == NULL)
+		return -EINVAL;
+
+	eventdev = rte_event_pmd_allocate(name,
+			 pci_dev->device.numa_node);
+	if (eventdev == NULL)
+		return -ENOMEM;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		eventdev->data->dev_private =
+				rte_zmalloc_socket(
+						"eventdev private structure",
+						private_data_size,
+						RTE_CACHE_LINE_SIZE,
+						rte_socket_id());
+
+		if (eventdev->data->dev_private == NULL)
+			rte_panic("Cannot allocate memzone for private "
+					"device data");
+	}
+
+	eventdev->dev = &pci_dev->device;
+
+	/* Invoke PMD device initialization function */
+	retval = devinit(eventdev);
+	if (retval == 0)
+		return 0;
+
+	RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)"
+			" failed", pci_drv->driver.name,
+			(unsigned int) pci_dev->id.vendor_id,
+			(unsigned int) pci_dev->id.device_id);
+
+	rte_event_pmd_release(eventdev);
+
+	return -ENXIO;
+}
 
 /**
  * @internal
-- 
2.13.6

Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

[Jerin Jacob]

On Sat, Jun 13, 2020 at 2:56 AM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
>
> The DLB hardware does not conform exactly to the eventdev interface.
> 1) It has a limit on the number of queues that may be linked to a port.
> 2) Some ports a further restricted to a maximum of 1 linked queue.
> 3) It does not (currently) have the ability to carry the flow_id as part
> of the event (QE) payload.
>
> Due to the above, we would like to propose the following enhancements.


Thanks, McDaniel, Good to see new HW PMD for eventdev.

+ Ray and Neil.

Hello McDaniel,
I assume this patchset is for v20.08. It is adding new elements in
pubic structures. Have you checked the ABI breakage?

I will review the rest of the series if there is NO ABI breakage as we
can not have the ABI breakage 20.08 version.


ABI validator
~~~~~~~~~~~~~~
1. meson build
2.  Compile and install known stable abi libs i.e ToT.
         DESTDIR=$PWD/install-meson-stable ninja -C build install
     Compile and install with patches to be verified.
         DESTDIR=$PWD/install-meson-new ninja -C build install
3. Gen ABI for both
        devtools/gen-abi.sh install-meson-stable
        devtools/gen-abi.sh install-meson-new
4. Run abi checker
        devtools/check-abi.sh install-meson-stable install-meson-new


DPDK_ABI_REF_DIR=/build/dpdk/reference/ DPDK_ABI_REF_VERSION=v20.02
./devtools/test-meson-builds.sh
DPDK_ABI_REF_DIR - needs an absolute path, for reasons that are still
unclear to me.
DPDK_ABI_REF_VERSION - you need to use the last DPDK release.

>
> 1) Add new fields to the rte_event_dev_info struct. These fields allow
> the device to advertize its capabilities so that applications can take
> the appropriate actions based on those capabilities.
>
>     struct rte_event_dev_info {
>         uint32_t max_event_port_links;
>         /**< Maximum number of queues that can be linked to a single event
>          * port by this device.
>          */
>
>         uint8_t max_single_link_event_port_queue_pairs;
>         /**< Maximum number of event ports and queues that are optimized for
>          * (and only capable of) single-link configurations supported by this
>          * device. These ports and queues are not accounted for in
>          * max_event_ports or max_event_queues.
>          */
>     }
>
> 2) Add a new field to the rte_event_dev_config struct. This field allows the
> application to specify how many of its ports are limited to a single link,
> or will be used in single link mode.
>
>     /** Event device configuration structure */
>     struct rte_event_dev_config {
>         uint8_t nb_single_link_event_port_queues;
>         /**< Number of event ports and queues that will be singly-linked to
>          * each other. These are a subset of the overall event ports and
>          * queues; this value cannot exceed *nb_event_ports* or
>          * *nb_event_queues*. If the device has ports and queues that are
>          * optimized for single-link usage, this field is a hint for how many
>          * to allocate; otherwise, regular event ports and queues can be used.
>          */
>     }
>
> 3) Replace the dedicated implicit_release_disabled field with a bit field
> of explicit port capabilities. The implicit_release_disable functionality
> is assiged to one bit, and a port-is-single-link-only  attribute is
> assigned to other, with the remaining bits available for future assignment.
>
>         * Event port configuration bitmap flags */
>         #define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>         /**< Configure the port not to release outstanding events in
>          * rte_event_dev_dequeue_burst(). If set, all events received through
>          * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>          * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>          * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>          */
>         #define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>
>         /**< This event port links only to a single event queue.
>          *
>          *  @see rte_event_port_setup(), rte_event_port_link()
>          */
>
>         #define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>         /**
>          * The implicit release disable attribute of the port
>          */
>
>         struct rte_event_port_conf {
>                 uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>         }
>
> 4) Add UMWAIT/UMONITOR bit to rte_cpuflags
>
> 5) Added a new API that is useful for probing PCI devices.
>
>         /**
>          * @internal
>          * Wrapper for use by pci drivers as a .probe function to attach to a event
>          * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>          * the name.
>          */
>         static inline int
>         rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>                                     struct rte_pci_device *pci_dev,
>                                     size_t private_data_size,
>                                     eventdev_pmd_pci_callback_t devinit,
>                                     const char *name);
>
> Change-Id: I4cf00015296e2b3feca9886895765554730594be
> Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> ---
>  app/test-eventdev/evt_common.h                     |  1 +
>  app/test-eventdev/test_order_atq.c                 |  4 ++
>  app/test-eventdev/test_order_common.c              |  6 ++-
>  app/test-eventdev/test_order_queue.c               |  4 ++
>  app/test-eventdev/test_perf_atq.c                  |  1 +
>  app/test-eventdev/test_perf_queue.c                |  1 +
>  app/test-eventdev/test_pipeline_atq.c              |  1 +
>  app/test-eventdev/test_pipeline_queue.c            |  1 +
>  app/test/test_eventdev.c                           |  4 +-
>  drivers/event/dpaa2/dpaa2_eventdev.c               |  2 +-
>  drivers/event/octeontx/ssovf_evdev.c               |  2 +-
>  drivers/event/skeleton/skeleton_eventdev.c         |  2 +-
>  drivers/event/sw/sw_evdev.c                        |  5 +-
>  drivers/event/sw/sw_evdev_selftest.c               |  9 ++--
>  .../eventdev_pipeline/pipeline_worker_generic.c    |  8 ++-
>  examples/eventdev_pipeline/pipeline_worker_tx.c    |  3 ++
>  examples/l2fwd-event/l2fwd_event_generic.c         |  5 +-
>  examples/l2fwd-event/l2fwd_event_internal_port.c   |  5 +-
>  examples/l3fwd/l3fwd_event_generic.c               |  5 +-
>  examples/l3fwd/l3fwd_event_internal_port.c         |  5 +-
>  lib/librte_eal/x86/include/rte_cpuflags.h          |  1 +
>  lib/librte_eal/x86/rte_cpuflags.c                  |  1 +
>  lib/librte_eventdev/rte_event_eth_tx_adapter.c     |  2 +-
>  lib/librte_eventdev/rte_eventdev.c                 | 62 +++++++++++++++++++---
>  lib/librte_eventdev/rte_eventdev.h                 | 51 +++++++++++++++---
>  lib/librte_eventdev/rte_eventdev_pmd_pci.h         | 54 +++++++++++++++++++
>  26 files changed, 208 insertions(+), 37 deletions(-)
>
> diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
> index f9d7378d3..120c27b33 100644
> --- a/app/test-eventdev/evt_common.h
> +++ b/app/test-eventdev/evt_common.h
> @@ -169,6 +169,7 @@ evt_configure_eventdev(struct evt_options *opt, uint8_t nb_queues,
>                         .dequeue_timeout_ns = opt->deq_tmo_nsec,
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 0,
>                         .nb_events_limit  = info.max_num_events,
>                         .nb_event_queue_flows = opt->nb_flows,
>                         .nb_event_port_dequeue_depth =
> diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
> index 3366cfce9..8246b96f0 100644
> --- a/app/test-eventdev/test_order_atq.c
> +++ b/app/test-eventdev/test_order_atq.c
> @@ -34,6 +34,8 @@ order_atq_worker(void *arg)
>                         continue;
>                 }
>
> +               ev.flow_id = ev.mbuf->udata64;
> +
>                 if (ev.sub_event_type == 0) { /* stage 0 from producer */
>                         order_atq_process_stage_0(&ev);
>                         while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
> @@ -68,6 +70,8 @@ order_atq_worker_burst(void *arg)
>                 }
>
>                 for (i = 0; i < nb_rx; i++) {
> +                       ev[i].flow_id = ev[i].mbuf->udata64;
> +
>                         if (ev[i].sub_event_type == 0) { /*stage 0 */
>                                 order_atq_process_stage_0(&ev[i]);
>                         } else if (ev[i].sub_event_type == 1) { /* stage 1 */
> diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
> index 4190f9ade..c6fcd0509 100644
> --- a/app/test-eventdev/test_order_common.c
> +++ b/app/test-eventdev/test_order_common.c
> @@ -49,6 +49,7 @@ order_producer(void *arg)
>                 const uint32_t flow = (uintptr_t)m % nb_flows;
>                 /* Maintain seq number per flow */
>                 m->seqn = producer_flow_seq[flow]++;
> +               m->udata64 = flow;
>
>                 ev.flow_id = flow;
>                 ev.mbuf = m;
> @@ -318,10 +319,11 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>                 opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
>
>         /* port configuration */
> -       const struct rte_event_port_conf p_conf = {
> +       struct rte_event_port_conf p_conf = {
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         /* setup one port per worker, linking to all queues */
> @@ -351,6 +353,8 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>         p->queue_id = 0;
>         p->t = t;
>
> +       p_conf.new_event_threshold /= 2;
> +
>         ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
>         if (ret) {
>                 evt_err("failed to setup producer port %d", port);
> diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
> index 495efd92f..a0a2187a2 100644
> --- a/app/test-eventdev/test_order_queue.c
> +++ b/app/test-eventdev/test_order_queue.c
> @@ -34,6 +34,8 @@ order_queue_worker(void *arg)
>                         continue;
>                 }
>
> +               ev.flow_id = ev.mbuf->udata64;
> +
>                 if (ev.queue_id == 0) { /* from ordered queue */
>                         order_queue_process_stage_0(&ev);
>                         while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
> @@ -68,6 +70,8 @@ order_queue_worker_burst(void *arg)
>                 }
>
>                 for (i = 0; i < nb_rx; i++) {
> +                       ev[i].flow_id = ev[i].mbuf->udata64;
> +
>                         if (ev[i].queue_id == 0) { /* from ordered queue */
>                                 order_queue_process_stage_0(&ev[i]);
>                         } else if (ev[i].queue_id == 1) {/* from atomic queue */
> diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
> index 8fd51004e..10846f202 100644
> --- a/app/test-eventdev/test_perf_atq.c
> +++ b/app/test-eventdev/test_perf_atq.c
> @@ -204,6 +204,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues,
> diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
> index f4ea3a795..a0119da60 100644
> --- a/app/test-eventdev/test_perf_queue.c
> +++ b/app/test-eventdev/test_perf_queue.c
> @@ -219,6 +219,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */,
> diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c
> index 8e8686c14..a95ec0aa5 100644
> --- a/app/test-eventdev/test_pipeline_atq.c
> +++ b/app/test-eventdev/test_pipeline_atq.c
> @@ -356,6 +356,7 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                 .dequeue_depth = opt->wkr_deq_dep,
>                 .enqueue_depth = info.max_event_port_dequeue_depth,
>                 .new_event_threshold = info.max_num_events,
> +               .event_port_cfg = 0,
>         };
>
>         if (!t->internal_port)
> diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c
> index 7bebac34f..30817dc78 100644
> --- a/app/test-eventdev/test_pipeline_queue.c
> +++ b/app/test-eventdev/test_pipeline_queue.c
> @@ -379,6 +379,7 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = info.max_event_port_dequeue_depth,
>                         .new_event_threshold = info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         if (!t->internal_port) {
> diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
> index 43ccb1ce9..62019c185 100644
> --- a/app/test/test_eventdev.c
> +++ b/app/test/test_eventdev.c
> @@ -559,10 +559,10 @@ test_eventdev_port_setup(void)
>         if (!(info.event_dev_cap &
>               RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>                 pconf.enqueue_depth = info.max_event_port_enqueue_depth;
> -               pconf.disable_implicit_release = 1;
> +               pconf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>                 ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf);
>                 TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret);
> -               pconf.disable_implicit_release = 0;
> +               pconf.event_port_cfg = 0;
>         }
>
>         ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports,
> diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
> index a196ad4c6..8568bfcfc 100644
> --- a/drivers/event/dpaa2/dpaa2_eventdev.c
> +++ b/drivers/event/dpaa2/dpaa2_eventdev.c
> @@ -537,7 +537,7 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>                 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
>         port_conf->enqueue_depth =
>                 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static int
> diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
> index 1b1a5d939..99c0b2efb 100644
> --- a/drivers/event/octeontx/ssovf_evdev.c
> +++ b/drivers/event/octeontx/ssovf_evdev.c
> @@ -224,7 +224,7 @@ ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = edev->max_num_events;
>         port_conf->dequeue_depth = 1;
>         port_conf->enqueue_depth = 1;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static void
> diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
> index c889220e0..37d569b8c 100644
> --- a/drivers/event/skeleton/skeleton_eventdev.c
> +++ b/drivers/event/skeleton/skeleton_eventdev.c
> @@ -209,7 +209,7 @@ skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = 32 * 1024;
>         port_conf->dequeue_depth = 16;
>         port_conf->enqueue_depth = 16;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static void
> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
> index fb8e8bebb..0b3dd9c1c 100644
> --- a/drivers/event/sw/sw_evdev.c
> +++ b/drivers/event/sw/sw_evdev.c
> @@ -175,7 +175,8 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
>         }
>
>         p->inflight_max = conf->new_event_threshold;
> -       p->implicit_release = !conf->disable_implicit_release;
> +       p->implicit_release = !(conf->event_port_cfg &
> +                               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>
>         /* check if ring exists, same as rx_worker above */
>         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
> @@ -508,7 +509,7 @@ sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = 1024;
>         port_conf->dequeue_depth = 16;
>         port_conf->enqueue_depth = 16;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static int
> diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
> index 38c21fa0f..a78d6cd0d 100644
> --- a/drivers/event/sw/sw_evdev_selftest.c
> +++ b/drivers/event/sw/sw_evdev_selftest.c
> @@ -172,7 +172,7 @@ create_ports(struct test *t, int num_ports)
>                         .new_event_threshold = 1024,
>                         .dequeue_depth = 32,
>                         .enqueue_depth = 64,
> -                       .disable_implicit_release = 0,
> +                       .event_port_cfg = 0,
>         };
>         if (num_ports > MAX_PORTS)
>                 return -1;
> @@ -1227,7 +1227,7 @@ port_reconfig_credits(struct test *t)
>                                 .new_event_threshold = 128,
>                                 .dequeue_depth = 32,
>                                 .enqueue_depth = 64,
> -                               .disable_implicit_release = 0,
> +                               .event_port_cfg = 0,
>                 };
>                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>                         printf("%d Error setting up port\n", __LINE__);
> @@ -1317,7 +1317,7 @@ port_single_lb_reconfig(struct test *t)
>                 .new_event_threshold = 128,
>                 .dequeue_depth = 32,
>                 .enqueue_depth = 64,
> -               .disable_implicit_release = 0,
> +               .event_port_cfg = 0,
>         };
>         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>                 printf("%d Error setting up port\n", __LINE__);
> @@ -3079,7 +3079,8 @@ worker_loopback(struct test *t, uint8_t disable_implicit_release)
>          * only be initialized once - and this needs to be set for multiple runs
>          */
>         conf.new_event_threshold = 512;
> -       conf.disable_implicit_release = disable_implicit_release;
> +       conf.event_port_cfg = disable_implicit_release ?
> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>
>         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
>                 printf("Error setting up RX port\n");
> diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
> index 42ff4eeb9..a091da3ba 100644
> --- a/examples/eventdev_pipeline/pipeline_worker_generic.c
> +++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
> @@ -129,6 +129,7 @@ setup_eventdev_generic(struct worker_data *worker_data)
>         struct rte_event_dev_config config = {
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 1,
>                         .nb_events_limit  = 4096,
>                         .nb_event_queue_flows = 1024,
>                         .nb_event_port_dequeue_depth = 128,
> @@ -138,12 +139,13 @@ setup_eventdev_generic(struct worker_data *worker_data)
>                         .dequeue_depth = cdata.worker_cq_depth,
>                         .enqueue_depth = 64,
>                         .new_event_threshold = 4096,
> +                       .event_port_cfg = 0,
>         };
>         struct rte_event_queue_conf wkr_q_conf = {
>                         .schedule_type = cdata.queue_type,
>                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
>                         .nb_atomic_flows = 1024,
> -               .nb_atomic_order_sequences = 1024,
> +                       .nb_atomic_order_sequences = 1024,
>         };
>         struct rte_event_queue_conf tx_q_conf = {
>                         .priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
> @@ -167,7 +169,8 @@ setup_eventdev_generic(struct worker_data *worker_data)
>         disable_implicit_release = (dev_info.event_dev_cap &
>                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
>
> -       wkr_p_conf.disable_implicit_release = disable_implicit_release;
> +       wkr_p_conf.event_port_cfg = disable_implicit_release ?
> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>
>         if (dev_info.max_num_events < config.nb_events_limit)
>                 config.nb_events_limit = dev_info.max_num_events;
> @@ -417,6 +420,7 @@ init_adapters(uint16_t nb_ports)
>                 .dequeue_depth = cdata.worker_cq_depth,
>                 .enqueue_depth = 64,
>                 .new_event_threshold = 4096,
> +               .event_port_cfg = 0,
>         };
>
>         if (adptr_p_conf.new_event_threshold > dev_info.max_num_events)
> diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
> index 55bb2f762..e8a9652aa 100644
> --- a/examples/eventdev_pipeline/pipeline_worker_tx.c
> +++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
> @@ -436,6 +436,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>         struct rte_event_dev_config config = {
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 0,
>                         .nb_events_limit  = 4096,
>                         .nb_event_queue_flows = 1024,
>                         .nb_event_port_dequeue_depth = 128,
> @@ -445,6 +446,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>                         .dequeue_depth = cdata.worker_cq_depth,
>                         .enqueue_depth = 64,
>                         .new_event_threshold = 4096,
> +                       .event_port_cfg = 0,
>         };
>         struct rte_event_queue_conf wkr_q_conf = {
>                         .schedule_type = cdata.queue_type,
> @@ -746,6 +748,7 @@ init_adapters(uint16_t nb_ports)
>                 .dequeue_depth = cdata.worker_cq_depth,
>                 .enqueue_depth = 64,
>                 .new_event_threshold = 4096,
> +               .event_port_cfg = 0,
>         };
>
>         init_ports(nb_ports);
> diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
> index 2dc95e5f7..e01df0435 100644
> --- a/examples/l2fwd-event/l2fwd_event_generic.c
> +++ b/examples/l2fwd-event/l2fwd_event_generic.c
> @@ -126,8 +126,9 @@ l2fwd_event_port_setup_generic(struct l2fwd_resources *rsrc)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>         evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
> diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
> index 63d57b46c..f54327b4f 100644
> --- a/examples/l2fwd-event/l2fwd_event_internal_port.c
> +++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
> @@ -123,8 +123,9 @@ l2fwd_event_port_setup_internal_port(struct l2fwd_resources *rsrc)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>                                                                 event_p_id++) {
> diff --git a/examples/l3fwd/l3fwd_event_generic.c b/examples/l3fwd/l3fwd_event_generic.c
> index f8c98435d..409a4107e 100644
> --- a/examples/l3fwd/l3fwd_event_generic.c
> +++ b/examples/l3fwd/l3fwd_event_generic.c
> @@ -115,8 +115,9 @@ l3fwd_event_port_setup_generic(void)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>         evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
> diff --git a/examples/l3fwd/l3fwd_event_internal_port.c b/examples/l3fwd/l3fwd_event_internal_port.c
> index 03ac581d6..df410f10f 100644
> --- a/examples/l3fwd/l3fwd_event_internal_port.c
> +++ b/examples/l3fwd/l3fwd_event_internal_port.c
> @@ -113,8 +113,9 @@ l3fwd_event_port_setup_internal_port(void)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>                                                                 event_p_id++) {
> diff --git a/lib/librte_eal/x86/include/rte_cpuflags.h b/lib/librte_eal/x86/include/rte_cpuflags.h
> index c1d20364d..ab2c3b379 100644
> --- a/lib/librte_eal/x86/include/rte_cpuflags.h
> +++ b/lib/librte_eal/x86/include/rte_cpuflags.h
> @@ -130,6 +130,7 @@ enum rte_cpu_flag_t {
>         RTE_CPUFLAG_CLDEMOTE,               /**< Cache Line Demote */
>         RTE_CPUFLAG_MOVDIRI,                /**< Direct Store Instructions */
>         RTE_CPUFLAG_MOVDIR64B,              /**< Direct Store Instructions 64B */
> +       RTE_CPUFLAG_UMWAIT,                 /**< UMONITOR/UMWAIT */
>         RTE_CPUFLAG_AVX512VP2INTERSECT,     /**< AVX512 Two Register Intersection */
>
>         /* The last item */
> diff --git a/lib/librte_eal/x86/rte_cpuflags.c b/lib/librte_eal/x86/rte_cpuflags.c
> index 30439e795..69ac0dbce 100644
> --- a/lib/librte_eal/x86/rte_cpuflags.c
> +++ b/lib/librte_eal/x86/rte_cpuflags.c
> @@ -137,6 +137,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
>         FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25)
>         FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27)
>         FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28)
> +        FEAT_DEF(UMWAIT, 0x00000007, 0, RTE_REG_ECX, 5)
>         FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8)
>  };
>
> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> index bb21dc407..8a72256de 100644
> --- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> @@ -286,7 +286,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id,
>                 return ret;
>         }
>
> -       pc->disable_implicit_release = 0;
> +       pc->event_port_cfg = 0;
>         ret = rte_event_port_setup(dev_id, port_id, pc);
>         if (ret) {
>                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
> diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
> index 82c177c73..4955ab1a0 100644
> --- a/lib/librte_eventdev/rte_eventdev.c
> +++ b/lib/librte_eventdev/rte_eventdev.c
> @@ -437,9 +437,29 @@ rte_event_dev_configure(uint8_t dev_id,
>                                         dev_id);
>                 return -EINVAL;
>         }
> -       if (dev_conf->nb_event_queues > info.max_event_queues) {
> -               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d",
> -               dev_id, dev_conf->nb_event_queues, info.max_event_queues);
> +       if (dev_conf->nb_event_queues > info.max_event_queues +
> +                       info.max_single_link_event_port_queue_pairs) {
> +               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d + max_single_link_event_port_queue_pairs=%d",
> +                                dev_id, dev_conf->nb_event_queues,
> +                                info.max_event_queues,
> +                                info.max_single_link_event_port_queue_pairs);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_event_queues -
> +                       dev_conf->nb_single_link_event_port_queues >
> +                       info.max_event_queues) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_queues=%d - nb_single_link_event_port_queues=%d > max_event_queues=%d",
> +                                dev_id, dev_conf->nb_event_queues,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                info.max_event_queues);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_single_link_event_port_queues >
> +                       dev_conf->nb_event_queues) {
> +               RTE_EDEV_LOG_ERR("dev%d nb_single_link_event_port_queues=%d > nb_event_queues=%d",
> +                                dev_id,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                dev_conf->nb_event_queues);
>                 return -EINVAL;
>         }
>
> @@ -448,9 +468,31 @@ rte_event_dev_configure(uint8_t dev_id,
>                 RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id);
>                 return -EINVAL;
>         }
> -       if (dev_conf->nb_event_ports > info.max_event_ports) {
> -               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d",
> -               dev_id, dev_conf->nb_event_ports, info.max_event_ports);
> +       if (dev_conf->nb_event_ports > info.max_event_ports +
> +                       info.max_single_link_event_port_queue_pairs) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports=%d + max_single_link_event_port_queue_pairs=%d",
> +                                dev_id, dev_conf->nb_event_ports,
> +                                info.max_event_ports,
> +                                info.max_single_link_event_port_queue_pairs);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_event_ports -
> +                       dev_conf->nb_single_link_event_port_queues
> +                       > info.max_event_ports) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d - nb_single_link_event_port_queues=%d > max_event_ports=%d",
> +                                dev_id, dev_conf->nb_event_ports,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                info.max_event_ports);
> +               return -EINVAL;
> +       }
> +
> +       if (dev_conf->nb_single_link_event_port_queues >
> +           dev_conf->nb_event_ports) {
> +               RTE_EDEV_LOG_ERR(
> +                                "dev%d nb_single_link_event_port_queues=%d > nb_event_ports=%d",
> +                                dev_id,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                dev_conf->nb_event_ports);
>                 return -EINVAL;
>         }
>
> @@ -737,7 +779,8 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>                 return -EINVAL;
>         }
>
> -       if (port_conf && port_conf->disable_implicit_release &&
> +       if (port_conf &&
> +           (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL) &&
>             !(dev->data->event_dev_cap &
>               RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>                 RTE_EDEV_LOG_ERR(
> @@ -809,6 +852,7 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>                         uint32_t *attr_value)
>  {
>         struct rte_eventdev *dev;
> +       uint32_t config;
>
>         if (!attr_value)
>                 return -EINVAL;
> @@ -830,6 +874,10 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>         case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD:
>                 *attr_value = dev->data->ports_cfg[port_id].new_event_threshold;
>                 break;
> +       case RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE:
> +               config = dev->data->ports_cfg[port_id].event_port_cfg;
> +               *attr_value = !!(config & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
> +               break;
>         default:
>                 return -EINVAL;
>         };
> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
> index 7dc832353..7f7a8a275 100644
> --- a/lib/librte_eventdev/rte_eventdev.h
> +++ b/lib/librte_eventdev/rte_eventdev.h
> @@ -291,6 +291,13 @@ struct rte_event;
>   * single queue to each port or map a single queue to many port.
>   */
>
> +#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)
> +/**< Event device is capable of carrying the flow ID from the enqueued
> + * event to the dequeued event. If the flag is set, the dequeued event's flow
> + * ID matches the corresponding enqueued event's flow ID. If the flag is not
> + * set, the dequeued event's flow ID field is uninitialized.
> + */
> +
>  /* Event device priority levels */
>  #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
>  /**< Highest priority expressed across eventdev subsystem
> @@ -380,6 +387,10 @@ struct rte_event_dev_info {
>          * event port by this device.
>          * A device that does not support bulk enqueue will set this as 1.
>          */
> +       uint32_t max_event_port_links;
> +       /**< Maximum number of queues that can be linked to a single event
> +        * port by this device.
> +        */
>         int32_t max_num_events;
>         /**< A *closed system* event dev has a limit on the number of events it
>          * can manage at a time. An *open system* event dev does not have a
> @@ -387,6 +398,12 @@ struct rte_event_dev_info {
>          */
>         uint32_t event_dev_cap;
>         /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/
> +       uint8_t max_single_link_event_port_queue_pairs;
> +       /**< Maximum number of event ports and queues that are optimized for
> +        * (and only capable of) single-link configurations supported by this
> +        * device. These ports and queues are not accounted for in
> +        * max_event_ports or max_event_queues.
> +        */
>  };
>
>  /**
> @@ -494,6 +511,14 @@ struct rte_event_dev_config {
>          */
>         uint32_t event_dev_cfg;
>         /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/
> +       uint8_t nb_single_link_event_port_queues;
> +       /**< Number of event ports and queues that will be singly-linked to
> +        * each other. These are a subset of the overall event ports and
> +        * queues; this value cannot exceed *nb_event_ports* or
> +        * *nb_event_queues*. If the device has ports and queues that are
> +        * optimized for single-link usage, this field is a hint for how many
> +        * to allocate; otherwise, regular event ports and queues can be used.
> +        */
>  };
>
>  /**
> @@ -671,6 +696,20 @@ rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
>
>  /* Event port specific APIs */
>
> +/* Event port configuration bitmap flags */
> +#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
> +/**< Configure the port not to release outstanding events in
> + * rte_event_dev_dequeue_burst(). If set, all events received through
> + * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
> + * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
> + * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
> + */
> +#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
> +/**< This event port links only to a single event queue.
> + *
> + *  @see rte_event_port_setup(), rte_event_port_link()
> + */
> +
>  /** Event port configuration structure */
>  struct rte_event_port_conf {
>         int32_t new_event_threshold;
> @@ -698,13 +737,7 @@ struct rte_event_port_conf {
>          * which previously supplied to rte_event_dev_configure().
>          * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable.
>          */
> -       uint8_t disable_implicit_release;
> -       /**< Configure the port not to release outstanding events in
> -        * rte_event_dev_dequeue_burst(). If true, all events received through
> -        * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
> -        * RTE_EVENT_OP_FORWARD. Must be false when the device is not
> -        * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
> -        */
> +       uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>  };
>
>  /**
> @@ -769,6 +802,10 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>   * The new event threshold of the port
>   */
>  #define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2
> +/**
> + * The implicit release disable attribute of the port
> + */
> +#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>
>  /**
>   * Get an attribute from a port.
> diff --git a/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> index 443cd38c2..157299983 100644
> --- a/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> +++ b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> @@ -88,6 +88,60 @@ rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv,
>         return -ENXIO;
>  }
>
> +/**
> + * @internal
> + * Wrapper for use by pci drivers as a .probe function to attach to a event
> + * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
> + * the name.
> + */
> +static inline int
> +rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
> +                           struct rte_pci_device *pci_dev,
> +                           size_t private_data_size,
> +                           eventdev_pmd_pci_callback_t devinit,
> +                           const char *name)
> +{
> +       struct rte_eventdev *eventdev;
> +
> +       int retval;
> +
> +       if (devinit == NULL)
> +               return -EINVAL;
> +
> +       eventdev = rte_event_pmd_allocate(name,
> +                        pci_dev->device.numa_node);
> +       if (eventdev == NULL)
> +               return -ENOMEM;
> +
> +       if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> +               eventdev->data->dev_private =
> +                               rte_zmalloc_socket(
> +                                               "eventdev private structure",
> +                                               private_data_size,
> +                                               RTE_CACHE_LINE_SIZE,
> +                                               rte_socket_id());
> +
> +               if (eventdev->data->dev_private == NULL)
> +                       rte_panic("Cannot allocate memzone for private "
> +                                       "device data");
> +       }
> +
> +       eventdev->dev = &pci_dev->device;
> +
> +       /* Invoke PMD device initialization function */
> +       retval = devinit(eventdev);
> +       if (retval == 0)
> +               return 0;
> +
> +       RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)"
> +                       " failed", pci_drv->driver.name,
> +                       (unsigned int) pci_dev->id.vendor_id,
> +                       (unsigned int) pci_dev->id.device_id);
> +
> +       rte_event_pmd_release(eventdev);
> +
> +       return -ENXIO;
> +}
>
>  /**
>   * @internal
> --
> 2.13.6
>

Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

[Mattias Rönnblom]

On 2020-06-13 05:59, Jerin Jacob wrote:
> On Sat, Jun 13, 2020 at 2:56 AM McDaniel, Timothy
> <timothy.mcdaniel@intel.com> wrote:
>> The DLB hardware does not conform exactly to the eventdev interface.
>> 1) It has a limit on the number of queues that may be linked to a port.
>> 2) Some ports a further restricted to a maximum of 1 linked queue.
>> 3) It does not (currently) have the ability to carry the flow_id as part
>> of the event (QE) payload.
>>
>> Due to the above, we would like to propose the following enhancements.
>
> Thanks, McDaniel, Good to see new HW PMD for eventdev.
>
> + Ray and Neil.
>
> Hello McDaniel,
> I assume this patchset is for v20.08. It is adding new elements in
> pubic structures. Have you checked the ABI breakage?
>
> I will review the rest of the series if there is NO ABI breakage as we
> can not have the ABI breakage 20.08 version.
>
>
> ABI validator
> ~~~~~~~~~~~~~~
> 1. meson build
> 2.  Compile and install known stable abi libs i.e ToT.
>           DESTDIR=$PWD/install-meson-stable ninja -C build install
>       Compile and install with patches to be verified.
>           DESTDIR=$PWD/install-meson-new ninja -C build install
> 3. Gen ABI for both
>          devtools/gen-abi.sh install-meson-stable
>          devtools/gen-abi.sh install-meson-new
> 4. Run abi checker
>          devtools/check-abi.sh install-meson-stable install-meson-new
>
>
> DPDK_ABI_REF_DIR=/build/dpdk/reference/ DPDK_ABI_REF_VERSION=v20.02
> ./devtools/test-meson-builds.sh
> DPDK_ABI_REF_DIR - needs an absolute path, for reasons that are still
> unclear to me.
> DPDK_ABI_REF_VERSION - you need to use the last DPDK release.
>
>> 1) Add new fields to the rte_event_dev_info struct. These fields allow
>> the device to advertize its capabilities so that applications can take
>> the appropriate actions based on those capabilities.
>>
>>      struct rte_event_dev_info {
>>          uint32_t max_event_port_links;
>>          /**< Maximum number of queues that can be linked to a single event
>>           * port by this device.
>>           */
>>
>>          uint8_t max_single_link_event_port_queue_pairs;
>>          /**< Maximum number of event ports and queues that are optimized for
>>           * (and only capable of) single-link configurations supported by this
>>           * device. These ports and queues are not accounted for in
>>           * max_event_ports or max_event_queues.
>>           */
>>      }
>>
>> 2) Add a new field to the rte_event_dev_config struct. This field allows the
>> application to specify how many of its ports are limited to a single link,
>> or will be used in single link mode.
>>
>>      /** Event device configuration structure */
>>      struct rte_event_dev_config {
>>          uint8_t nb_single_link_event_port_queues;
>>          /**< Number of event ports and queues that will be singly-linked to
>>           * each other. These are a subset of the overall event ports and
>>           * queues; this value cannot exceed *nb_event_ports* or
>>           * *nb_event_queues*. If the device has ports and queues that are
>>           * optimized for single-link usage, this field is a hint for how many
>>           * to allocate; otherwise, regular event ports and queues can be used.
>>           */
>>      }
>>
>> 3) Replace the dedicated implicit_release_disabled field with a bit field
>> of explicit port capabilities. The implicit_release_disable functionality
>> is assiged to one bit, and a port-is-single-link-only  attribute is
>> assigned to other, with the remaining bits available for future assignment.
>>
>>          * Event port configuration bitmap flags */
>>          #define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>>          /**< Configure the port not to release outstanding events in
>>           * rte_event_dev_dequeue_burst(). If set, all events received through
>>           * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>>           * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>>           * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>>           */
>>          #define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>>
>>          /**< This event port links only to a single event queue.
>>           *
>>           *  @see rte_event_port_setup(), rte_event_port_link()
>>           */
>>
>>          #define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>>          /**
>>           * The implicit release disable attribute of the port
>>           */
>>
>>          struct rte_event_port_conf {
>>                  uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>>          }
>>
>> 4) Add UMWAIT/UMONITOR bit to rte_cpuflags
>>
>> 5) Added a new API that is useful for probing PCI devices.
>>
>>          /**
>>           * @internal
>>           * Wrapper for use by pci drivers as a .probe function to attach to a event
>>           * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>>           * the name.
>>           */
>>          static inline int
>>          rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>>                                      struct rte_pci_device *pci_dev,
>>                                      size_t private_data_size,
>>                                      eventdev_pmd_pci_callback_t devinit,
>>                                      const char *name);
>>
>> Change-Id: I4cf00015296e2b3feca9886895765554730594be
>> Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
>> ---
>>   app/test-eventdev/evt_common.h                     |  1 +
>>   app/test-eventdev/test_order_atq.c                 |  4 ++
>>   app/test-eventdev/test_order_common.c              |  6 ++-
>>   app/test-eventdev/test_order_queue.c               |  4 ++
>>   app/test-eventdev/test_perf_atq.c                  |  1 +
>>   app/test-eventdev/test_perf_queue.c                |  1 +
>>   app/test-eventdev/test_pipeline_atq.c              |  1 +
>>   app/test-eventdev/test_pipeline_queue.c            |  1 +
>>   app/test/test_eventdev.c                           |  4 +-
>>   drivers/event/dpaa2/dpaa2_eventdev.c               |  2 +-
>>   drivers/event/octeontx/ssovf_evdev.c               |  2 +-
>>   drivers/event/skeleton/skeleton_eventdev.c         |  2 +-
>>   drivers/event/sw/sw_evdev.c                        |  5 +-
>>   drivers/event/sw/sw_evdev_selftest.c               |  9 ++--
>>   .../eventdev_pipeline/pipeline_worker_generic.c    |  8 ++-
>>   examples/eventdev_pipeline/pipeline_worker_tx.c    |  3 ++
>>   examples/l2fwd-event/l2fwd_event_generic.c         |  5 +-
>>   examples/l2fwd-event/l2fwd_event_internal_port.c   |  5 +-
>>   examples/l3fwd/l3fwd_event_generic.c               |  5 +-
>>   examples/l3fwd/l3fwd_event_internal_port.c         |  5 +-
>>   lib/librte_eal/x86/include/rte_cpuflags.h          |  1 +
>>   lib/librte_eal/x86/rte_cpuflags.c                  |  1 +
>>   lib/librte_eventdev/rte_event_eth_tx_adapter.c     |  2 +-
>>   lib/librte_eventdev/rte_eventdev.c                 | 62 +++++++++++++++++++---
>>   lib/librte_eventdev/rte_eventdev.h                 | 51 +++++++++++++++---
>>   lib/librte_eventdev/rte_eventdev_pmd_pci.h         | 54 +++++++++++++++++++
>>   26 files changed, 208 insertions(+), 37 deletions(-)
>>
>> diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
>> index f9d7378d3..120c27b33 100644
>> --- a/app/test-eventdev/evt_common.h
>> +++ b/app/test-eventdev/evt_common.h
>> @@ -169,6 +169,7 @@ evt_configure_eventdev(struct evt_options *opt, uint8_t nb_queues,
>>                          .dequeue_timeout_ns = opt->deq_tmo_nsec,
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 0,
>>                          .nb_events_limit  = info.max_num_events,
>>                          .nb_event_queue_flows = opt->nb_flows,
>>                          .nb_event_port_dequeue_depth =
>> diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
>> index 3366cfce9..8246b96f0 100644
>> --- a/app/test-eventdev/test_order_atq.c
>> +++ b/app/test-eventdev/test_order_atq.c
>> @@ -34,6 +34,8 @@ order_atq_worker(void *arg)
>>                          continue;
>>                  }
>>
>> +               ev.flow_id = ev.mbuf->udata64;
>> +
>>                  if (ev.sub_event_type == 0) { /* stage 0 from producer */
>>                          order_atq_process_stage_0(&ev);
>>                          while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
>> @@ -68,6 +70,8 @@ order_atq_worker_burst(void *arg)
>>                  }
>>
>>                  for (i = 0; i < nb_rx; i++) {
>> +                       ev[i].flow_id = ev[i].mbuf->udata64;
>> +
>>                          if (ev[i].sub_event_type == 0) { /*stage 0 */
>>                                  order_atq_process_stage_0(&ev[i]);
>>                          } else if (ev[i].sub_event_type == 1) { /* stage 1 */
>> diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
>> index 4190f9ade..c6fcd0509 100644
>> --- a/app/test-eventdev/test_order_common.c
>> +++ b/app/test-eventdev/test_order_common.c
>> @@ -49,6 +49,7 @@ order_producer(void *arg)
>>                  const uint32_t flow = (uintptr_t)m % nb_flows;
>>                  /* Maintain seq number per flow */
>>                  m->seqn = producer_flow_seq[flow]++;
>> +               m->udata64 = flow;
>>
>>                  ev.flow_id = flow;
>>                  ev.mbuf = m;
>> @@ -318,10 +319,11 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>>                  opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
>>
>>          /* port configuration */
>> -       const struct rte_event_port_conf p_conf = {
>> +       struct rte_event_port_conf p_conf = {
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          /* setup one port per worker, linking to all queues */
>> @@ -351,6 +353,8 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>>          p->queue_id = 0;
>>          p->t = t;
>>
>> +       p_conf.new_event_threshold /= 2;
>> +
>>          ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
>>          if (ret) {
>>                  evt_err("failed to setup producer port %d", port);
>> diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
>> index 495efd92f..a0a2187a2 100644
>> --- a/app/test-eventdev/test_order_queue.c
>> +++ b/app/test-eventdev/test_order_queue.c
>> @@ -34,6 +34,8 @@ order_queue_worker(void *arg)
>>                          continue;
>>                  }
>>
>> +               ev.flow_id = ev.mbuf->udata64;
>> +
>>                  if (ev.queue_id == 0) { /* from ordered queue */
>>                          order_queue_process_stage_0(&ev);
>>                          while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
>> @@ -68,6 +70,8 @@ order_queue_worker_burst(void *arg)
>>                  }
>>
>>                  for (i = 0; i < nb_rx; i++) {
>> +                       ev[i].flow_id = ev[i].mbuf->udata64;
>> +
>>                          if (ev[i].queue_id == 0) { /* from ordered queue */
>>                                  order_queue_process_stage_0(&ev[i]);
>>                          } else if (ev[i].queue_id == 1) {/* from atomic queue */
>> diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
>> index 8fd51004e..10846f202 100644
>> --- a/app/test-eventdev/test_perf_atq.c
>> +++ b/app/test-eventdev/test_perf_atq.c
>> @@ -204,6 +204,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues,
>> diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
>> index f4ea3a795..a0119da60 100644
>> --- a/app/test-eventdev/test_perf_queue.c
>> +++ b/app/test-eventdev/test_perf_queue.c
>> @@ -219,6 +219,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */,
>> diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c
>> index 8e8686c14..a95ec0aa5 100644
>> --- a/app/test-eventdev/test_pipeline_atq.c
>> +++ b/app/test-eventdev/test_pipeline_atq.c
>> @@ -356,6 +356,7 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                  .dequeue_depth = opt->wkr_deq_dep,
>>                  .enqueue_depth = info.max_event_port_dequeue_depth,
>>                  .new_event_threshold = info.max_num_events,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          if (!t->internal_port)
>> diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c
>> index 7bebac34f..30817dc78 100644
>> --- a/app/test-eventdev/test_pipeline_queue.c
>> +++ b/app/test-eventdev/test_pipeline_queue.c
>> @@ -379,6 +379,7 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          if (!t->internal_port) {
>> diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
>> index 43ccb1ce9..62019c185 100644
>> --- a/app/test/test_eventdev.c
>> +++ b/app/test/test_eventdev.c
>> @@ -559,10 +559,10 @@ test_eventdev_port_setup(void)
>>          if (!(info.event_dev_cap &
>>                RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>>                  pconf.enqueue_depth = info.max_event_port_enqueue_depth;
>> -               pconf.disable_implicit_release = 1;
>> +               pconf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>                  ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf);
>>                  TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret);
>> -               pconf.disable_implicit_release = 0;
>> +               pconf.event_port_cfg = 0;
>>          }
>>
>>          ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports,
>> diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
>> index a196ad4c6..8568bfcfc 100644
>> --- a/drivers/event/dpaa2/dpaa2_eventdev.c
>> +++ b/drivers/event/dpaa2/dpaa2_eventdev.c
>> @@ -537,7 +537,7 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>                  DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
>>          port_conf->enqueue_depth =
>>                  DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static int
>> diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
>> index 1b1a5d939..99c0b2efb 100644
>> --- a/drivers/event/octeontx/ssovf_evdev.c
>> +++ b/drivers/event/octeontx/ssovf_evdev.c
>> @@ -224,7 +224,7 @@ ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = edev->max_num_events;
>>          port_conf->dequeue_depth = 1;
>>          port_conf->enqueue_depth = 1;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static void
>> diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
>> index c889220e0..37d569b8c 100644
>> --- a/drivers/event/skeleton/skeleton_eventdev.c
>> +++ b/drivers/event/skeleton/skeleton_eventdev.c
>> @@ -209,7 +209,7 @@ skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = 32 * 1024;
>>          port_conf->dequeue_depth = 16;
>>          port_conf->enqueue_depth = 16;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static void
>> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
>> index fb8e8bebb..0b3dd9c1c 100644
>> --- a/drivers/event/sw/sw_evdev.c
>> +++ b/drivers/event/sw/sw_evdev.c
>> @@ -175,7 +175,8 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
>>          }
>>
>>          p->inflight_max = conf->new_event_threshold;
>> -       p->implicit_release = !conf->disable_implicit_release;
>> +       p->implicit_release = !(conf->event_port_cfg &
>> +                               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>>
>>          /* check if ring exists, same as rx_worker above */
>>          snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
>> @@ -508,7 +509,7 @@ sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = 1024;
>>          port_conf->dequeue_depth = 16;
>>          port_conf->enqueue_depth = 16;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static int
>> diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
>> index 38c21fa0f..a78d6cd0d 100644
>> --- a/drivers/event/sw/sw_evdev_selftest.c
>> +++ b/drivers/event/sw/sw_evdev_selftest.c
>> @@ -172,7 +172,7 @@ create_ports(struct test *t, int num_ports)
>>                          .new_event_threshold = 1024,
>>                          .dequeue_depth = 32,
>>                          .enqueue_depth = 64,
>> -                       .disable_implicit_release = 0,
>> +                       .event_port_cfg = 0,
>>          };
>>          if (num_ports > MAX_PORTS)
>>                  return -1;
>> @@ -1227,7 +1227,7 @@ port_reconfig_credits(struct test *t)
>>                                  .new_event_threshold = 128,
>>                                  .dequeue_depth = 32,
>>                                  .enqueue_depth = 64,
>> -                               .disable_implicit_release = 0,
>> +                               .event_port_cfg = 0,
>>                  };
>>                  if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>>                          printf("%d Error setting up port\n", __LINE__);
>> @@ -1317,7 +1317,7 @@ port_single_lb_reconfig(struct test *t)
>>                  .new_event_threshold = 128,
>>                  .dequeue_depth = 32,
>>                  .enqueue_depth = 64,
>> -               .disable_implicit_release = 0,
>> +               .event_port_cfg = 0,
>>          };
>>          if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>>                  printf("%d Error setting up port\n", __LINE__);
>> @@ -3079,7 +3079,8 @@ worker_loopback(struct test *t, uint8_t disable_implicit_release)
>>           * only be initialized once - and this needs to be set for multiple runs
>>           */
>>          conf.new_event_threshold = 512;
>> -       conf.disable_implicit_release = disable_implicit_release;
>> +       conf.event_port_cfg = disable_implicit_release ?
>> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>>
>>          if (rte_event_port_setup(evdev, 0, &conf) < 0) {
>>                  printf("Error setting up RX port\n");
>> diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
>> index 42ff4eeb9..a091da3ba 100644
>> --- a/examples/eventdev_pipeline/pipeline_worker_generic.c
>> +++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
>> @@ -129,6 +129,7 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>          struct rte_event_dev_config config = {
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 1,
>>                          .nb_events_limit  = 4096,
>>                          .nb_event_queue_flows = 1024,
>>                          .nb_event_port_dequeue_depth = 128,
>> @@ -138,12 +139,13 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>                          .dequeue_depth = cdata.worker_cq_depth,
>>                          .enqueue_depth = 64,
>>                          .new_event_threshold = 4096,
>> +                       .event_port_cfg = 0,


No need to set this value; it's guaranteed to be 0 anyways. You might 
argue you do it for readability, but two other fields of that struct is 
already implicitly initialized.


This would apply to other of your changes as well.


>>          };
>>          struct rte_event_queue_conf wkr_q_conf = {
>>                          .schedule_type = cdata.queue_type,
>>                          .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
>>                          .nb_atomic_flows = 1024,
>> -               .nb_atomic_order_sequences = 1024,
>> +                       .nb_atomic_order_sequences = 1024,
>>          };
>>          struct rte_event_queue_conf tx_q_conf = {
>>                          .priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
>> @@ -167,7 +169,8 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>          disable_implicit_release = (dev_info.event_dev_cap &
>>                          RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
>>
>> -       wkr_p_conf.disable_implicit_release = disable_implicit_release;
>> +       wkr_p_conf.event_port_cfg = disable_implicit_release ?
>> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>>
>>          if (dev_info.max_num_events < config.nb_events_limit)
>>                  config.nb_events_limit = dev_info.max_num_events;
>> @@ -417,6 +420,7 @@ init_adapters(uint16_t nb_ports)
>>                  .dequeue_depth = cdata.worker_cq_depth,
>>                  .enqueue_depth = 64,
>>                  .new_event_threshold = 4096,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          if (adptr_p_conf.new_event_threshold > dev_info.max_num_events)
>> diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
>> index 55bb2f762..e8a9652aa 100644
>> --- a/examples/eventdev_pipeline/pipeline_worker_tx.c
>> +++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
>> @@ -436,6 +436,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>>          struct rte_event_dev_config config = {
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 0,
>>                          .nb_events_limit  = 4096,
>>                          .nb_event_queue_flows = 1024,
>>                          .nb_event_port_dequeue_depth = 128,
>> @@ -445,6 +446,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>>                          .dequeue_depth = cdata.worker_cq_depth,
>>                          .enqueue_depth = 64,
>>                          .new_event_threshold = 4096,
>> +                       .event_port_cfg = 0,
>>          };
>>          struct rte_event_queue_conf wkr_q_conf = {
>>                          .schedule_type = cdata.queue_type,
>> @@ -746,6 +748,7 @@ init_adapters(uint16_t nb_ports)
>>                  .dequeue_depth = cdata.worker_cq_depth,
>>                  .enqueue_depth = 64,
>>                  .new_event_threshold = 4096,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          init_ports(nb_ports);
>> diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
>> index 2dc95e5f7..e01df0435 100644
>> --- a/examples/l2fwd-event/l2fwd_event_generic.c
>> +++ b/examples/l2fwd-event/l2fwd_event_generic.c
>> @@ -126,8 +126,9 @@ l2fwd_event_port_setup_generic(struct l2fwd_resources *rsrc)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>          evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>> diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
>> index 63d57b46c..f54327b4f 100644
>> --- a/examples/l2fwd-event/l2fwd_event_internal_port.c
>> +++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
>> @@ -123,8 +123,9 @@ l2fwd_event_port_setup_internal_port(struct l2fwd_resources *rsrc)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>>                                                                  event_p_id++) {
>> diff --git a/examples/l3fwd/l3fwd_event_generic.c b/examples/l3fwd/l3fwd_event_generic.c
>> index f8c98435d..409a4107e 100644
>> --- a/examples/l3fwd/l3fwd_event_generic.c
>> +++ b/examples/l3fwd/l3fwd_event_generic.c
>> @@ -115,8 +115,9 @@ l3fwd_event_port_setup_generic(void)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>          evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>> diff --git a/examples/l3fwd/l3fwd_event_internal_port.c b/examples/l3fwd/l3fwd_event_internal_port.c
>> index 03ac581d6..df410f10f 100644
>> --- a/examples/l3fwd/l3fwd_event_internal_port.c
>> +++ b/examples/l3fwd/l3fwd_event_internal_port.c
>> @@ -113,8 +113,9 @@ l3fwd_event_port_setup_internal_port(void)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>>                                                                  event_p_id++) {
>> diff --git a/lib/librte_eal/x86/include/rte_cpuflags.h b/lib/librte_eal/x86/include/rte_cpuflags.h
>> index c1d20364d..ab2c3b379 100644
>> --- a/lib/librte_eal/x86/include/rte_cpuflags.h
>> +++ b/lib/librte_eal/x86/include/rte_cpuflags.h
>> @@ -130,6 +130,7 @@ enum rte_cpu_flag_t {
>>          RTE_CPUFLAG_CLDEMOTE,               /**< Cache Line Demote */
>>          RTE_CPUFLAG_MOVDIRI,                /**< Direct Store Instructions */
>>          RTE_CPUFLAG_MOVDIR64B,              /**< Direct Store Instructions 64B */
>> +       RTE_CPUFLAG_UMWAIT,                 /**< UMONITOR/UMWAIT */
>>          RTE_CPUFLAG_AVX512VP2INTERSECT,     /**< AVX512 Two Register Intersection */
>>
>>          /* The last item */
>> diff --git a/lib/librte_eal/x86/rte_cpuflags.c b/lib/librte_eal/x86/rte_cpuflags.c
>> index 30439e795..69ac0dbce 100644
>> --- a/lib/librte_eal/x86/rte_cpuflags.c
>> +++ b/lib/librte_eal/x86/rte_cpuflags.c
>> @@ -137,6 +137,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
>>          FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25)
>>          FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27)
>>          FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28)
>> +        FEAT_DEF(UMWAIT, 0x00000007, 0, RTE_REG_ECX, 5)
>>          FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8)
>>   };
>>
>> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> index bb21dc407..8a72256de 100644
>> --- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> @@ -286,7 +286,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id,
>>                  return ret;
>>          }
>>
>> -       pc->disable_implicit_release = 0;
>> +       pc->event_port_cfg = 0;
>>          ret = rte_event_port_setup(dev_id, port_id, pc);
>>          if (ret) {
>>                  RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
>> diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
>> index 82c177c73..4955ab1a0 100644
>> --- a/lib/librte_eventdev/rte_eventdev.c
>> +++ b/lib/librte_eventdev/rte_eventdev.c
>> @@ -437,9 +437,29 @@ rte_event_dev_configure(uint8_t dev_id,
>>                                          dev_id);
>>                  return -EINVAL;
>>          }
>> -       if (dev_conf->nb_event_queues > info.max_event_queues) {
>> -               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d",
>> -               dev_id, dev_conf->nb_event_queues, info.max_event_queues);
>> +       if (dev_conf->nb_event_queues > info.max_event_queues +
>> +                       info.max_single_link_event_port_queue_pairs) {
>> +               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d + max_single_link_event_port_queue_pairs=%d",
>> +                                dev_id, dev_conf->nb_event_queues,
>> +                                info.max_event_queues,
>> +                                info.max_single_link_event_port_queue_pairs);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_event_queues -
>> +                       dev_conf->nb_single_link_event_port_queues >
>> +                       info.max_event_queues) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_queues=%d - nb_single_link_event_port_queues=%d > max_event_queues=%d",
>> +                                dev_id, dev_conf->nb_event_queues,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                info.max_event_queues);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_single_link_event_port_queues >
>> +                       dev_conf->nb_event_queues) {
>> +               RTE_EDEV_LOG_ERR("dev%d nb_single_link_event_port_queues=%d > nb_event_queues=%d",
>> +                                dev_id,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                dev_conf->nb_event_queues);
>>                  return -EINVAL;
>>          }
>>
>> @@ -448,9 +468,31 @@ rte_event_dev_configure(uint8_t dev_id,
>>                  RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id);
>>                  return -EINVAL;
>>          }
>> -       if (dev_conf->nb_event_ports > info.max_event_ports) {
>> -               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d",
>> -               dev_id, dev_conf->nb_event_ports, info.max_event_ports);
>> +       if (dev_conf->nb_event_ports > info.max_event_ports +
>> +                       info.max_single_link_event_port_queue_pairs) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports=%d + max_single_link_event_port_queue_pairs=%d",
>> +                                dev_id, dev_conf->nb_event_ports,
>> +                                info.max_event_ports,
>> +                                info.max_single_link_event_port_queue_pairs);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_event_ports -
>> +                       dev_conf->nb_single_link_event_port_queues
>> +                       > info.max_event_ports) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d - nb_single_link_event_port_queues=%d > max_event_ports=%d",
>> +                                dev_id, dev_conf->nb_event_ports,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                info.max_event_ports);
>> +               return -EINVAL;
>> +       }
>> +
>> +       if (dev_conf->nb_single_link_event_port_queues >
>> +           dev_conf->nb_event_ports) {
>> +               RTE_EDEV_LOG_ERR(
>> +                                "dev%d nb_single_link_event_port_queues=%d > nb_event_ports=%d",
>> +                                dev_id,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                dev_conf->nb_event_ports);
>>                  return -EINVAL;
>>          }
>>
>> @@ -737,7 +779,8 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>>                  return -EINVAL;
>>          }
>>
>> -       if (port_conf && port_conf->disable_implicit_release &&
>> +       if (port_conf &&
>> +           (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL) &&
>>              !(dev->data->event_dev_cap &
>>                RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>>                  RTE_EDEV_LOG_ERR(
>> @@ -809,6 +852,7 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>>                          uint32_t *attr_value)
>>   {
>>          struct rte_eventdev *dev;
>> +       uint32_t config;
>>
>>          if (!attr_value)
>>                  return -EINVAL;
>> @@ -830,6 +874,10 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>>          case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD:
>>                  *attr_value = dev->data->ports_cfg[port_id].new_event_threshold;
>>                  break;
>> +       case RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE:
>> +               config = dev->data->ports_cfg[port_id].event_port_cfg;
>> +               *attr_value = !!(config & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>> +               break;
>>          default:
>>                  return -EINVAL;
>>          };
>> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
>> index 7dc832353..7f7a8a275 100644
>> --- a/lib/librte_eventdev/rte_eventdev.h
>> +++ b/lib/librte_eventdev/rte_eventdev.h
>> @@ -291,6 +291,13 @@ struct rte_event;
>>    * single queue to each port or map a single queue to many port.
>>    */
>>
>> +#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)
>> +/**< Event device is capable of carrying the flow ID from the enqueued
>> + * event to the dequeued event. If the flag is set, the dequeued event's flow
>> + * ID matches the corresponding enqueued event's flow ID. If the flag is not
>> + * set, the dequeued event's flow ID field is uninitialized.
>> + */
>> +

The dequeued event's value should be undefined, to let an implementation 
overwrite an existing value. Replace "is capable of carrying" with 
"carries".


Is "maintain" better than "carry"? Or "preserve". I don't know.

>>   /* Event device priority levels */
>>   #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
>>   /**< Highest priority expressed across eventdev subsystem
>> @@ -380,6 +387,10 @@ struct rte_event_dev_info {
>>           * event port by this device.
>>           * A device that does not support bulk enqueue will set this as 1.
>>           */
>> +       uint32_t max_event_port_links;
>> +       /**< Maximum number of queues that can be linked to a single event
>> +        * port by this device.
>> +        */


Eventdev API supports 255 queues, so you should use an uint8_t here.


>>          int32_t max_num_events;
>>          /**< A *closed system* event dev has a limit on the number of events it
>>           * can manage at a time. An *open system* event dev does not have a
>> @@ -387,6 +398,12 @@ struct rte_event_dev_info {
>>           */
>>          uint32_t event_dev_cap;
>>          /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/
>> +       uint8_t max_single_link_event_port_queue_pairs;
>> +       /**< Maximum number of event ports and queues that are optimized for
>> +        * (and only capable of) single-link configurations supported by this
>> +        * device. These ports and queues are not accounted for in
>> +        * max_event_ports or max_event_queues.
>> +        */
>>   };
>>
>>   /**
>> @@ -494,6 +511,14 @@ struct rte_event_dev_config {
>>           */
>>          uint32_t event_dev_cfg;
>>          /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/
>> +       uint8_t nb_single_link_event_port_queues;
>> +       /**< Number of event ports and queues that will be singly-linked to
>> +        * each other. These are a subset of the overall event ports and
>> +        * queues; this value cannot exceed *nb_event_ports* or
>> +        * *nb_event_queues*. If the device has ports and queues that are
>> +        * optimized for single-link usage, this field is a hint for how many
>> +        * to allocate; otherwise, regular event ports and queues can be used.
>> +        */
>>   };
>>
>>   /**
>> @@ -671,6 +696,20 @@ rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
>>
>>   /* Event port specific APIs */
>>
>> +/* Event port configuration bitmap flags */
>> +#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>> +/**< Configure the port not to release outstanding events in
>> + * rte_event_dev_dequeue_burst(). If set, all events received through
>> + * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>> + * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>> + * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>> + */
>> +#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>> +/**< This event port links only to a single event queue.
>> + *
>> + *  @see rte_event_port_setup(), rte_event_port_link()
>> + */
>> +
>>   /** Event port configuration structure */
>>   struct rte_event_port_conf {
>>          int32_t new_event_threshold;
>> @@ -698,13 +737,7 @@ struct rte_event_port_conf {
>>           * which previously supplied to rte_event_dev_configure().
>>           * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable.
>>           */
>> -       uint8_t disable_implicit_release;
>> -       /**< Configure the port not to release outstanding events in
>> -        * rte_event_dev_dequeue_burst(). If true, all events received through
>> -        * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>> -        * RTE_EVENT_OP_FORWARD. Must be false when the device is not
>> -        * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>> -        */
>> +       uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>>   };
>>
>>   /**
>> @@ -769,6 +802,10 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>>    * The new event threshold of the port
>>    */
>>   #define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2
>> +/**
>> + * The implicit release disable attribute of the port
>> + */
>> +#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>>
>>   /**
>>    * Get an attribute from a port.
>> diff --git a/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> index 443cd38c2..157299983 100644
>> --- a/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> +++ b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> @@ -88,6 +88,60 @@ rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv,
>>          return -ENXIO;
>>   }
>>
>> +/**
>> + * @internal
>> + * Wrapper for use by pci drivers as a .probe function to attach to a event
>> + * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>> + * the name.
>> + */
>> +static inline int
>> +rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>> +                           struct rte_pci_device *pci_dev,
>> +                           size_t private_data_size,
>> +                           eventdev_pmd_pci_callback_t devinit,
>> +                           const char *name)
>> +{
>> +       struct rte_eventdev *eventdev;
>> +
>> +       int retval;
>> +
>> +       if (devinit == NULL)
>> +               return -EINVAL;
>> +
>> +       eventdev = rte_event_pmd_allocate(name,
>> +                        pci_dev->device.numa_node);
>> +       if (eventdev == NULL)
>> +               return -ENOMEM;
>> +
>> +       if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
>> +               eventdev->data->dev_private =
>> +                               rte_zmalloc_socket(
>> +                                               "eventdev private structure",
>> +                                               private_data_size,
>> +                                               RTE_CACHE_LINE_SIZE,
>> +                                               rte_socket_id());
>> +
>> +               if (eventdev->data->dev_private == NULL)
>> +                       rte_panic("Cannot allocate memzone for private "
>> +                                       "device data");
>> +       }
>> +
>> +       eventdev->dev = &pci_dev->device;
>> +
>> +       /* Invoke PMD device initialization function */
>> +       retval = devinit(eventdev);
>> +       if (retval == 0)
>> +               return 0;
>> +
>> +       RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)"
>> +                       " failed", pci_drv->driver.name,
>> +                       (unsigned int) pci_dev->id.vendor_id,
>> +                       (unsigned int) pci_dev->id.device_id);
>> +
>> +       rte_event_pmd_release(eventdev);
>> +
>> +       return -ENXIO;
>> +}
>>
>>   /**
>>    * @internal
>> --
>> 2.13.6
>>

Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

[McDaniel, Timothy]

Hello Mattias,

Thank you for your review comments. I will incorporate the changes you have suggested in V2 of the patchset, which I am currently working on.

Thanks,
Tim

-----Original Message-----
From: Mattias Rönnblom <mattias.ronnblom@ericsson.com> 
Sent: Saturday, June 13, 2020 5:44 AM
To: Jerin Jacob <jerinjacobk@gmail.com>; McDaniel, Timothy <timothy.mcdaniel@intel.com>
Cc: Jerin Jacob <jerinj@marvell.com>; dpdk-dev <dev@dpdk.org>; Eads, Gage <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
Subject: Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

On 2020-06-13 05:59, Jerin Jacob wrote:
> On Sat, Jun 13, 2020 at 2:56 AM McDaniel, Timothy
> <timothy.mcdaniel@intel.com> wrote:
>> The DLB hardware does not conform exactly to the eventdev interface.
>> 1) It has a limit on the number of queues that may be linked to a port.
>> 2) Some ports a further restricted to a maximum of 1 linked queue.
>> 3) It does not (currently) have the ability to carry the flow_id as part
>> of the event (QE) payload.
>>
>> Due to the above, we would like to propose the following enhancements.
>
> Thanks, McDaniel, Good to see new HW PMD for eventdev.
>
> + Ray and Neil.
>
> Hello McDaniel,
> I assume this patchset is for v20.08. It is adding new elements in
> pubic structures. Have you checked the ABI breakage?
>
> I will review the rest of the series if there is NO ABI breakage as we
> can not have the ABI breakage 20.08 version.
>
>
> ABI validator
> ~~~~~~~~~~~~~~
> 1. meson build
> 2.  Compile and install known stable abi libs i.e ToT.
>           DESTDIR=$PWD/install-meson-stable ninja -C build install
>       Compile and install with patches to be verified.
>           DESTDIR=$PWD/install-meson-new ninja -C build install
> 3. Gen ABI for both
>          devtools/gen-abi.sh install-meson-stable
>          devtools/gen-abi.sh install-meson-new
> 4. Run abi checker
>          devtools/check-abi.sh install-meson-stable install-meson-new
>
>
> DPDK_ABI_REF_DIR=/build/dpdk/reference/ DPDK_ABI_REF_VERSION=v20.02
> ./devtools/test-meson-builds.sh
> DPDK_ABI_REF_DIR - needs an absolute path, for reasons that are still
> unclear to me.
> DPDK_ABI_REF_VERSION - you need to use the last DPDK release.
>
>> 1) Add new fields to the rte_event_dev_info struct. These fields allow
>> the device to advertize its capabilities so that applications can take
>> the appropriate actions based on those capabilities.
>>
>>      struct rte_event_dev_info {
>>          uint32_t max_event_port_links;
>>          /**< Maximum number of queues that can be linked to a single event
>>           * port by this device.
>>           */
>>
>>          uint8_t max_single_link_event_port_queue_pairs;
>>          /**< Maximum number of event ports and queues that are optimized for
>>           * (and only capable of) single-link configurations supported by this
>>           * device. These ports and queues are not accounted for in
>>           * max_event_ports or max_event_queues.
>>           */
>>      }
>>
>> 2) Add a new field to the rte_event_dev_config struct. This field allows the
>> application to specify how many of its ports are limited to a single link,
>> or will be used in single link mode.
>>
>>      /** Event device configuration structure */
>>      struct rte_event_dev_config {
>>          uint8_t nb_single_link_event_port_queues;
>>          /**< Number of event ports and queues that will be singly-linked to
>>           * each other. These are a subset of the overall event ports and
>>           * queues; this value cannot exceed *nb_event_ports* or
>>           * *nb_event_queues*. If the device has ports and queues that are
>>           * optimized for single-link usage, this field is a hint for how many
>>           * to allocate; otherwise, regular event ports and queues can be used.
>>           */
>>      }
>>
>> 3) Replace the dedicated implicit_release_disabled field with a bit field
>> of explicit port capabilities. The implicit_release_disable functionality
>> is assiged to one bit, and a port-is-single-link-only  attribute is
>> assigned to other, with the remaining bits available for future assignment.
>>
>>          * Event port configuration bitmap flags */
>>          #define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>>          /**< Configure the port not to release outstanding events in
>>           * rte_event_dev_dequeue_burst(). If set, all events received through
>>           * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>>           * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>>           * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>>           */
>>          #define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>>
>>          /**< This event port links only to a single event queue.
>>           *
>>           *  @see rte_event_port_setup(), rte_event_port_link()
>>           */
>>
>>          #define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>>          /**
>>           * The implicit release disable attribute of the port
>>           */
>>
>>          struct rte_event_port_conf {
>>                  uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>>          }
>>
>> 4) Add UMWAIT/UMONITOR bit to rte_cpuflags
>>
>> 5) Added a new API that is useful for probing PCI devices.
>>
>>          /**
>>           * @internal
>>           * Wrapper for use by pci drivers as a .probe function to attach to a event
>>           * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>>           * the name.
>>           */
>>          static inline int
>>          rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>>                                      struct rte_pci_device *pci_dev,
>>                                      size_t private_data_size,
>>                                      eventdev_pmd_pci_callback_t devinit,
>>                                      const char *name);
>>
>> Change-Id: I4cf00015296e2b3feca9886895765554730594be
>> Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
>> ---
>>   app/test-eventdev/evt_common.h                     |  1 +
>>   app/test-eventdev/test_order_atq.c                 |  4 ++
>>   app/test-eventdev/test_order_common.c              |  6 ++-
>>   app/test-eventdev/test_order_queue.c               |  4 ++
>>   app/test-eventdev/test_perf_atq.c                  |  1 +
>>   app/test-eventdev/test_perf_queue.c                |  1 +
>>   app/test-eventdev/test_pipeline_atq.c              |  1 +
>>   app/test-eventdev/test_pipeline_queue.c            |  1 +
>>   app/test/test_eventdev.c                           |  4 +-
>>   drivers/event/dpaa2/dpaa2_eventdev.c               |  2 +-
>>   drivers/event/octeontx/ssovf_evdev.c               |  2 +-
>>   drivers/event/skeleton/skeleton_eventdev.c         |  2 +-
>>   drivers/event/sw/sw_evdev.c                        |  5 +-
>>   drivers/event/sw/sw_evdev_selftest.c               |  9 ++--
>>   .../eventdev_pipeline/pipeline_worker_generic.c    |  8 ++-
>>   examples/eventdev_pipeline/pipeline_worker_tx.c    |  3 ++
>>   examples/l2fwd-event/l2fwd_event_generic.c         |  5 +-
>>   examples/l2fwd-event/l2fwd_event_internal_port.c   |  5 +-
>>   examples/l3fwd/l3fwd_event_generic.c               |  5 +-
>>   examples/l3fwd/l3fwd_event_internal_port.c         |  5 +-
>>   lib/librte_eal/x86/include/rte_cpuflags.h          |  1 +
>>   lib/librte_eal/x86/rte_cpuflags.c                  |  1 +
>>   lib/librte_eventdev/rte_event_eth_tx_adapter.c     |  2 +-
>>   lib/librte_eventdev/rte_eventdev.c                 | 62 +++++++++++++++++++---
>>   lib/librte_eventdev/rte_eventdev.h                 | 51 +++++++++++++++---
>>   lib/librte_eventdev/rte_eventdev_pmd_pci.h         | 54 +++++++++++++++++++
>>   26 files changed, 208 insertions(+), 37 deletions(-)
>>
>> diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
>> index f9d7378d3..120c27b33 100644
>> --- a/app/test-eventdev/evt_common.h
>> +++ b/app/test-eventdev/evt_common.h
>> @@ -169,6 +169,7 @@ evt_configure_eventdev(struct evt_options *opt, uint8_t nb_queues,
>>                          .dequeue_timeout_ns = opt->deq_tmo_nsec,
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 0,
>>                          .nb_events_limit  = info.max_num_events,
>>                          .nb_event_queue_flows = opt->nb_flows,
>>                          .nb_event_port_dequeue_depth =
>> diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
>> index 3366cfce9..8246b96f0 100644
>> --- a/app/test-eventdev/test_order_atq.c
>> +++ b/app/test-eventdev/test_order_atq.c
>> @@ -34,6 +34,8 @@ order_atq_worker(void *arg)
>>                          continue;
>>                  }
>>
>> +               ev.flow_id = ev.mbuf->udata64;
>> +
>>                  if (ev.sub_event_type == 0) { /* stage 0 from producer */
>>                          order_atq_process_stage_0(&ev);
>>                          while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
>> @@ -68,6 +70,8 @@ order_atq_worker_burst(void *arg)
>>                  }
>>
>>                  for (i = 0; i < nb_rx; i++) {
>> +                       ev[i].flow_id = ev[i].mbuf->udata64;
>> +
>>                          if (ev[i].sub_event_type == 0) { /*stage 0 */
>>                                  order_atq_process_stage_0(&ev[i]);
>>                          } else if (ev[i].sub_event_type == 1) { /* stage 1 */
>> diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
>> index 4190f9ade..c6fcd0509 100644
>> --- a/app/test-eventdev/test_order_common.c
>> +++ b/app/test-eventdev/test_order_common.c
>> @@ -49,6 +49,7 @@ order_producer(void *arg)
>>                  const uint32_t flow = (uintptr_t)m % nb_flows;
>>                  /* Maintain seq number per flow */
>>                  m->seqn = producer_flow_seq[flow]++;
>> +               m->udata64 = flow;
>>
>>                  ev.flow_id = flow;
>>                  ev.mbuf = m;
>> @@ -318,10 +319,11 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>>                  opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
>>
>>          /* port configuration */
>> -       const struct rte_event_port_conf p_conf = {
>> +       struct rte_event_port_conf p_conf = {
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          /* setup one port per worker, linking to all queues */
>> @@ -351,6 +353,8 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>>          p->queue_id = 0;
>>          p->t = t;
>>
>> +       p_conf.new_event_threshold /= 2;
>> +
>>          ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
>>          if (ret) {
>>                  evt_err("failed to setup producer port %d", port);
>> diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
>> index 495efd92f..a0a2187a2 100644
>> --- a/app/test-eventdev/test_order_queue.c
>> +++ b/app/test-eventdev/test_order_queue.c
>> @@ -34,6 +34,8 @@ order_queue_worker(void *arg)
>>                          continue;
>>                  }
>>
>> +               ev.flow_id = ev.mbuf->udata64;
>> +
>>                  if (ev.queue_id == 0) { /* from ordered queue */
>>                          order_queue_process_stage_0(&ev);
>>                          while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
>> @@ -68,6 +70,8 @@ order_queue_worker_burst(void *arg)
>>                  }
>>
>>                  for (i = 0; i < nb_rx; i++) {
>> +                       ev[i].flow_id = ev[i].mbuf->udata64;
>> +
>>                          if (ev[i].queue_id == 0) { /* from ordered queue */
>>                                  order_queue_process_stage_0(&ev[i]);
>>                          } else if (ev[i].queue_id == 1) {/* from atomic queue */
>> diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
>> index 8fd51004e..10846f202 100644
>> --- a/app/test-eventdev/test_perf_atq.c
>> +++ b/app/test-eventdev/test_perf_atq.c
>> @@ -204,6 +204,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues,
>> diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
>> index f4ea3a795..a0119da60 100644
>> --- a/app/test-eventdev/test_perf_queue.c
>> +++ b/app/test-eventdev/test_perf_queue.c
>> @@ -219,6 +219,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = dev_info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */,
>> diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c
>> index 8e8686c14..a95ec0aa5 100644
>> --- a/app/test-eventdev/test_pipeline_atq.c
>> +++ b/app/test-eventdev/test_pipeline_atq.c
>> @@ -356,6 +356,7 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                  .dequeue_depth = opt->wkr_deq_dep,
>>                  .enqueue_depth = info.max_event_port_dequeue_depth,
>>                  .new_event_threshold = info.max_num_events,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          if (!t->internal_port)
>> diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c
>> index 7bebac34f..30817dc78 100644
>> --- a/app/test-eventdev/test_pipeline_queue.c
>> +++ b/app/test-eventdev/test_pipeline_queue.c
>> @@ -379,6 +379,7 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>>                          .dequeue_depth = opt->wkr_deq_dep,
>>                          .enqueue_depth = info.max_event_port_dequeue_depth,
>>                          .new_event_threshold = info.max_num_events,
>> +                       .event_port_cfg = 0,
>>          };
>>
>>          if (!t->internal_port) {
>> diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
>> index 43ccb1ce9..62019c185 100644
>> --- a/app/test/test_eventdev.c
>> +++ b/app/test/test_eventdev.c
>> @@ -559,10 +559,10 @@ test_eventdev_port_setup(void)
>>          if (!(info.event_dev_cap &
>>                RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>>                  pconf.enqueue_depth = info.max_event_port_enqueue_depth;
>> -               pconf.disable_implicit_release = 1;
>> +               pconf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>                  ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf);
>>                  TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret);
>> -               pconf.disable_implicit_release = 0;
>> +               pconf.event_port_cfg = 0;
>>          }
>>
>>          ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports,
>> diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
>> index a196ad4c6..8568bfcfc 100644
>> --- a/drivers/event/dpaa2/dpaa2_eventdev.c
>> +++ b/drivers/event/dpaa2/dpaa2_eventdev.c
>> @@ -537,7 +537,7 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>                  DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
>>          port_conf->enqueue_depth =
>>                  DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static int
>> diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
>> index 1b1a5d939..99c0b2efb 100644
>> --- a/drivers/event/octeontx/ssovf_evdev.c
>> +++ b/drivers/event/octeontx/ssovf_evdev.c
>> @@ -224,7 +224,7 @@ ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = edev->max_num_events;
>>          port_conf->dequeue_depth = 1;
>>          port_conf->enqueue_depth = 1;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static void
>> diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
>> index c889220e0..37d569b8c 100644
>> --- a/drivers/event/skeleton/skeleton_eventdev.c
>> +++ b/drivers/event/skeleton/skeleton_eventdev.c
>> @@ -209,7 +209,7 @@ skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = 32 * 1024;
>>          port_conf->dequeue_depth = 16;
>>          port_conf->enqueue_depth = 16;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static void
>> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
>> index fb8e8bebb..0b3dd9c1c 100644
>> --- a/drivers/event/sw/sw_evdev.c
>> +++ b/drivers/event/sw/sw_evdev.c
>> @@ -175,7 +175,8 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
>>          }
>>
>>          p->inflight_max = conf->new_event_threshold;
>> -       p->implicit_release = !conf->disable_implicit_release;
>> +       p->implicit_release = !(conf->event_port_cfg &
>> +                               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>>
>>          /* check if ring exists, same as rx_worker above */
>>          snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
>> @@ -508,7 +509,7 @@ sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>>          port_conf->new_event_threshold = 1024;
>>          port_conf->dequeue_depth = 16;
>>          port_conf->enqueue_depth = 16;
>> -       port_conf->disable_implicit_release = 0;
>> +       port_conf->event_port_cfg = 0;
>>   }
>>
>>   static int
>> diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
>> index 38c21fa0f..a78d6cd0d 100644
>> --- a/drivers/event/sw/sw_evdev_selftest.c
>> +++ b/drivers/event/sw/sw_evdev_selftest.c
>> @@ -172,7 +172,7 @@ create_ports(struct test *t, int num_ports)
>>                          .new_event_threshold = 1024,
>>                          .dequeue_depth = 32,
>>                          .enqueue_depth = 64,
>> -                       .disable_implicit_release = 0,
>> +                       .event_port_cfg = 0,
>>          };
>>          if (num_ports > MAX_PORTS)
>>                  return -1;
>> @@ -1227,7 +1227,7 @@ port_reconfig_credits(struct test *t)
>>                                  .new_event_threshold = 128,
>>                                  .dequeue_depth = 32,
>>                                  .enqueue_depth = 64,
>> -                               .disable_implicit_release = 0,
>> +                               .event_port_cfg = 0,
>>                  };
>>                  if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>>                          printf("%d Error setting up port\n", __LINE__);
>> @@ -1317,7 +1317,7 @@ port_single_lb_reconfig(struct test *t)
>>                  .new_event_threshold = 128,
>>                  .dequeue_depth = 32,
>>                  .enqueue_depth = 64,
>> -               .disable_implicit_release = 0,
>> +               .event_port_cfg = 0,
>>          };
>>          if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>>                  printf("%d Error setting up port\n", __LINE__);
>> @@ -3079,7 +3079,8 @@ worker_loopback(struct test *t, uint8_t disable_implicit_release)
>>           * only be initialized once - and this needs to be set for multiple runs
>>           */
>>          conf.new_event_threshold = 512;
>> -       conf.disable_implicit_release = disable_implicit_release;
>> +       conf.event_port_cfg = disable_implicit_release ?
>> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>>
>>          if (rte_event_port_setup(evdev, 0, &conf) < 0) {
>>                  printf("Error setting up RX port\n");
>> diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
>> index 42ff4eeb9..a091da3ba 100644
>> --- a/examples/eventdev_pipeline/pipeline_worker_generic.c
>> +++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
>> @@ -129,6 +129,7 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>          struct rte_event_dev_config config = {
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 1,
>>                          .nb_events_limit  = 4096,
>>                          .nb_event_queue_flows = 1024,
>>                          .nb_event_port_dequeue_depth = 128,
>> @@ -138,12 +139,13 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>                          .dequeue_depth = cdata.worker_cq_depth,
>>                          .enqueue_depth = 64,
>>                          .new_event_threshold = 4096,
>> +                       .event_port_cfg = 0,


No need to set this value; it's guaranteed to be 0 anyways. You might 
argue you do it for readability, but two other fields of that struct is 
already implicitly initialized.


This would apply to other of your changes as well.


>>          };
>>          struct rte_event_queue_conf wkr_q_conf = {
>>                          .schedule_type = cdata.queue_type,
>>                          .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
>>                          .nb_atomic_flows = 1024,
>> -               .nb_atomic_order_sequences = 1024,
>> +                       .nb_atomic_order_sequences = 1024,
>>          };
>>          struct rte_event_queue_conf tx_q_conf = {
>>                          .priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
>> @@ -167,7 +169,8 @@ setup_eventdev_generic(struct worker_data *worker_data)
>>          disable_implicit_release = (dev_info.event_dev_cap &
>>                          RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
>>
>> -       wkr_p_conf.disable_implicit_release = disable_implicit_release;
>> +       wkr_p_conf.event_port_cfg = disable_implicit_release ?
>> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>>
>>          if (dev_info.max_num_events < config.nb_events_limit)
>>                  config.nb_events_limit = dev_info.max_num_events;
>> @@ -417,6 +420,7 @@ init_adapters(uint16_t nb_ports)
>>                  .dequeue_depth = cdata.worker_cq_depth,
>>                  .enqueue_depth = 64,
>>                  .new_event_threshold = 4096,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          if (adptr_p_conf.new_event_threshold > dev_info.max_num_events)
>> diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
>> index 55bb2f762..e8a9652aa 100644
>> --- a/examples/eventdev_pipeline/pipeline_worker_tx.c
>> +++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
>> @@ -436,6 +436,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>>          struct rte_event_dev_config config = {
>>                          .nb_event_queues = nb_queues,
>>                          .nb_event_ports = nb_ports,
>> +                       .nb_single_link_event_port_queues = 0,
>>                          .nb_events_limit  = 4096,
>>                          .nb_event_queue_flows = 1024,
>>                          .nb_event_port_dequeue_depth = 128,
>> @@ -445,6 +446,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>>                          .dequeue_depth = cdata.worker_cq_depth,
>>                          .enqueue_depth = 64,
>>                          .new_event_threshold = 4096,
>> +                       .event_port_cfg = 0,
>>          };
>>          struct rte_event_queue_conf wkr_q_conf = {
>>                          .schedule_type = cdata.queue_type,
>> @@ -746,6 +748,7 @@ init_adapters(uint16_t nb_ports)
>>                  .dequeue_depth = cdata.worker_cq_depth,
>>                  .enqueue_depth = 64,
>>                  .new_event_threshold = 4096,
>> +               .event_port_cfg = 0,
>>          };
>>
>>          init_ports(nb_ports);
>> diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
>> index 2dc95e5f7..e01df0435 100644
>> --- a/examples/l2fwd-event/l2fwd_event_generic.c
>> +++ b/examples/l2fwd-event/l2fwd_event_generic.c
>> @@ -126,8 +126,9 @@ l2fwd_event_port_setup_generic(struct l2fwd_resources *rsrc)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>          evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>> diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
>> index 63d57b46c..f54327b4f 100644
>> --- a/examples/l2fwd-event/l2fwd_event_internal_port.c
>> +++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
>> @@ -123,8 +123,9 @@ l2fwd_event_port_setup_internal_port(struct l2fwd_resources *rsrc)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>>                                                                  event_p_id++) {
>> diff --git a/examples/l3fwd/l3fwd_event_generic.c b/examples/l3fwd/l3fwd_event_generic.c
>> index f8c98435d..409a4107e 100644
>> --- a/examples/l3fwd/l3fwd_event_generic.c
>> +++ b/examples/l3fwd/l3fwd_event_generic.c
>> @@ -115,8 +115,9 @@ l3fwd_event_port_setup_generic(void)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>          evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>> diff --git a/examples/l3fwd/l3fwd_event_internal_port.c b/examples/l3fwd/l3fwd_event_internal_port.c
>> index 03ac581d6..df410f10f 100644
>> --- a/examples/l3fwd/l3fwd_event_internal_port.c
>> +++ b/examples/l3fwd/l3fwd_event_internal_port.c
>> @@ -113,8 +113,9 @@ l3fwd_event_port_setup_internal_port(void)
>>          if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>>                  event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>>
>> -       event_p_conf.disable_implicit_release =
>> -               evt_rsrc->disable_implicit_release;
>> +       event_p_conf.event_port_cfg = 0;
>> +       if (evt_rsrc->disable_implicit_release)
>> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>>
>>          for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>>                                                                  event_p_id++) {
>> diff --git a/lib/librte_eal/x86/include/rte_cpuflags.h b/lib/librte_eal/x86/include/rte_cpuflags.h
>> index c1d20364d..ab2c3b379 100644
>> --- a/lib/librte_eal/x86/include/rte_cpuflags.h
>> +++ b/lib/librte_eal/x86/include/rte_cpuflags.h
>> @@ -130,6 +130,7 @@ enum rte_cpu_flag_t {
>>          RTE_CPUFLAG_CLDEMOTE,               /**< Cache Line Demote */
>>          RTE_CPUFLAG_MOVDIRI,                /**< Direct Store Instructions */
>>          RTE_CPUFLAG_MOVDIR64B,              /**< Direct Store Instructions 64B */
>> +       RTE_CPUFLAG_UMWAIT,                 /**< UMONITOR/UMWAIT */
>>          RTE_CPUFLAG_AVX512VP2INTERSECT,     /**< AVX512 Two Register Intersection */
>>
>>          /* The last item */
>> diff --git a/lib/librte_eal/x86/rte_cpuflags.c b/lib/librte_eal/x86/rte_cpuflags.c
>> index 30439e795..69ac0dbce 100644
>> --- a/lib/librte_eal/x86/rte_cpuflags.c
>> +++ b/lib/librte_eal/x86/rte_cpuflags.c
>> @@ -137,6 +137,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
>>          FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25)
>>          FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27)
>>          FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28)
>> +        FEAT_DEF(UMWAIT, 0x00000007, 0, RTE_REG_ECX, 5)
>>          FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8)
>>   };
>>
>> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> index bb21dc407..8a72256de 100644
>> --- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
>> @@ -286,7 +286,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id,
>>                  return ret;
>>          }
>>
>> -       pc->disable_implicit_release = 0;
>> +       pc->event_port_cfg = 0;
>>          ret = rte_event_port_setup(dev_id, port_id, pc);
>>          if (ret) {
>>                  RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
>> diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
>> index 82c177c73..4955ab1a0 100644
>> --- a/lib/librte_eventdev/rte_eventdev.c
>> +++ b/lib/librte_eventdev/rte_eventdev.c
>> @@ -437,9 +437,29 @@ rte_event_dev_configure(uint8_t dev_id,
>>                                          dev_id);
>>                  return -EINVAL;
>>          }
>> -       if (dev_conf->nb_event_queues > info.max_event_queues) {
>> -               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d",
>> -               dev_id, dev_conf->nb_event_queues, info.max_event_queues);
>> +       if (dev_conf->nb_event_queues > info.max_event_queues +
>> +                       info.max_single_link_event_port_queue_pairs) {
>> +               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d + max_single_link_event_port_queue_pairs=%d",
>> +                                dev_id, dev_conf->nb_event_queues,
>> +                                info.max_event_queues,
>> +                                info.max_single_link_event_port_queue_pairs);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_event_queues -
>> +                       dev_conf->nb_single_link_event_port_queues >
>> +                       info.max_event_queues) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_queues=%d - nb_single_link_event_port_queues=%d > max_event_queues=%d",
>> +                                dev_id, dev_conf->nb_event_queues,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                info.max_event_queues);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_single_link_event_port_queues >
>> +                       dev_conf->nb_event_queues) {
>> +               RTE_EDEV_LOG_ERR("dev%d nb_single_link_event_port_queues=%d > nb_event_queues=%d",
>> +                                dev_id,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                dev_conf->nb_event_queues);
>>                  return -EINVAL;
>>          }
>>
>> @@ -448,9 +468,31 @@ rte_event_dev_configure(uint8_t dev_id,
>>                  RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id);
>>                  return -EINVAL;
>>          }
>> -       if (dev_conf->nb_event_ports > info.max_event_ports) {
>> -               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d",
>> -               dev_id, dev_conf->nb_event_ports, info.max_event_ports);
>> +       if (dev_conf->nb_event_ports > info.max_event_ports +
>> +                       info.max_single_link_event_port_queue_pairs) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports=%d + max_single_link_event_port_queue_pairs=%d",
>> +                                dev_id, dev_conf->nb_event_ports,
>> +                                info.max_event_ports,
>> +                                info.max_single_link_event_port_queue_pairs);
>> +               return -EINVAL;
>> +       }
>> +       if (dev_conf->nb_event_ports -
>> +                       dev_conf->nb_single_link_event_port_queues
>> +                       > info.max_event_ports) {
>> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d - nb_single_link_event_port_queues=%d > max_event_ports=%d",
>> +                                dev_id, dev_conf->nb_event_ports,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                info.max_event_ports);
>> +               return -EINVAL;
>> +       }
>> +
>> +       if (dev_conf->nb_single_link_event_port_queues >
>> +           dev_conf->nb_event_ports) {
>> +               RTE_EDEV_LOG_ERR(
>> +                                "dev%d nb_single_link_event_port_queues=%d > nb_event_ports=%d",
>> +                                dev_id,
>> +                                dev_conf->nb_single_link_event_port_queues,
>> +                                dev_conf->nb_event_ports);
>>                  return -EINVAL;
>>          }
>>
>> @@ -737,7 +779,8 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>>                  return -EINVAL;
>>          }
>>
>> -       if (port_conf && port_conf->disable_implicit_release &&
>> +       if (port_conf &&
>> +           (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL) &&
>>              !(dev->data->event_dev_cap &
>>                RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>>                  RTE_EDEV_LOG_ERR(
>> @@ -809,6 +852,7 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>>                          uint32_t *attr_value)
>>   {
>>          struct rte_eventdev *dev;
>> +       uint32_t config;
>>
>>          if (!attr_value)
>>                  return -EINVAL;
>> @@ -830,6 +874,10 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>>          case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD:
>>                  *attr_value = dev->data->ports_cfg[port_id].new_event_threshold;
>>                  break;
>> +       case RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE:
>> +               config = dev->data->ports_cfg[port_id].event_port_cfg;
>> +               *attr_value = !!(config & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>> +               break;
>>          default:
>>                  return -EINVAL;
>>          };
>> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
>> index 7dc832353..7f7a8a275 100644
>> --- a/lib/librte_eventdev/rte_eventdev.h
>> +++ b/lib/librte_eventdev/rte_eventdev.h
>> @@ -291,6 +291,13 @@ struct rte_event;
>>    * single queue to each port or map a single queue to many port.
>>    */
>>
>> +#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)
>> +/**< Event device is capable of carrying the flow ID from the enqueued
>> + * event to the dequeued event. If the flag is set, the dequeued event's flow
>> + * ID matches the corresponding enqueued event's flow ID. If the flag is not
>> + * set, the dequeued event's flow ID field is uninitialized.
>> + */
>> +

The dequeued event's value should be undefined, to let an implementation 
overwrite an existing value. Replace "is capable of carrying" with 
"carries".


Is "maintain" better than "carry"? Or "preserve". I don't know.

>>   /* Event device priority levels */
>>   #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
>>   /**< Highest priority expressed across eventdev subsystem
>> @@ -380,6 +387,10 @@ struct rte_event_dev_info {
>>           * event port by this device.
>>           * A device that does not support bulk enqueue will set this as 1.
>>           */
>> +       uint32_t max_event_port_links;
>> +       /**< Maximum number of queues that can be linked to a single event
>> +        * port by this device.
>> +        */


Eventdev API supports 255 queues, so you should use an uint8_t here.


>>          int32_t max_num_events;
>>          /**< A *closed system* event dev has a limit on the number of events it
>>           * can manage at a time. An *open system* event dev does not have a
>> @@ -387,6 +398,12 @@ struct rte_event_dev_info {
>>           */
>>          uint32_t event_dev_cap;
>>          /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/
>> +       uint8_t max_single_link_event_port_queue_pairs;
>> +       /**< Maximum number of event ports and queues that are optimized for
>> +        * (and only capable of) single-link configurations supported by this
>> +        * device. These ports and queues are not accounted for in
>> +        * max_event_ports or max_event_queues.
>> +        */
>>   };
>>
>>   /**
>> @@ -494,6 +511,14 @@ struct rte_event_dev_config {
>>           */
>>          uint32_t event_dev_cfg;
>>          /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/
>> +       uint8_t nb_single_link_event_port_queues;
>> +       /**< Number of event ports and queues that will be singly-linked to
>> +        * each other. These are a subset of the overall event ports and
>> +        * queues; this value cannot exceed *nb_event_ports* or
>> +        * *nb_event_queues*. If the device has ports and queues that are
>> +        * optimized for single-link usage, this field is a hint for how many
>> +        * to allocate; otherwise, regular event ports and queues can be used.
>> +        */
>>   };
>>
>>   /**
>> @@ -671,6 +696,20 @@ rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
>>
>>   /* Event port specific APIs */
>>
>> +/* Event port configuration bitmap flags */
>> +#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>> +/**< Configure the port not to release outstanding events in
>> + * rte_event_dev_dequeue_burst(). If set, all events received through
>> + * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>> + * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>> + * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>> + */
>> +#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>> +/**< This event port links only to a single event queue.
>> + *
>> + *  @see rte_event_port_setup(), rte_event_port_link()
>> + */
>> +
>>   /** Event port configuration structure */
>>   struct rte_event_port_conf {
>>          int32_t new_event_threshold;
>> @@ -698,13 +737,7 @@ struct rte_event_port_conf {
>>           * which previously supplied to rte_event_dev_configure().
>>           * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable.
>>           */
>> -       uint8_t disable_implicit_release;
>> -       /**< Configure the port not to release outstanding events in
>> -        * rte_event_dev_dequeue_burst(). If true, all events received through
>> -        * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>> -        * RTE_EVENT_OP_FORWARD. Must be false when the device is not
>> -        * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>> -        */
>> +       uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>>   };
>>
>>   /**
>> @@ -769,6 +802,10 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>>    * The new event threshold of the port
>>    */
>>   #define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2
>> +/**
>> + * The implicit release disable attribute of the port
>> + */
>> +#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>>
>>   /**
>>    * Get an attribute from a port.
>> diff --git a/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> index 443cd38c2..157299983 100644
>> --- a/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> +++ b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
>> @@ -88,6 +88,60 @@ rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv,
>>          return -ENXIO;
>>   }
>>
>> +/**
>> + * @internal
>> + * Wrapper for use by pci drivers as a .probe function to attach to a event
>> + * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>> + * the name.
>> + */
>> +static inline int
>> +rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>> +                           struct rte_pci_device *pci_dev,
>> +                           size_t private_data_size,
>> +                           eventdev_pmd_pci_callback_t devinit,
>> +                           const char *name)
>> +{
>> +       struct rte_eventdev *eventdev;
>> +
>> +       int retval;
>> +
>> +       if (devinit == NULL)
>> +               return -EINVAL;
>> +
>> +       eventdev = rte_event_pmd_allocate(name,
>> +                        pci_dev->device.numa_node);
>> +       if (eventdev == NULL)
>> +               return -ENOMEM;
>> +
>> +       if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
>> +               eventdev->data->dev_private =
>> +                               rte_zmalloc_socket(
>> +                                               "eventdev private structure",
>> +                                               private_data_size,
>> +                                               RTE_CACHE_LINE_SIZE,
>> +                                               rte_socket_id());
>> +
>> +               if (eventdev->data->dev_private == NULL)
>> +                       rte_panic("Cannot allocate memzone for private "
>> +                                       "device data");
>> +       }
>> +
>> +       eventdev->dev = &pci_dev->device;
>> +
>> +       /* Invoke PMD device initialization function */
>> +       retval = devinit(eventdev);
>> +       if (retval == 0)
>> +               return 0;
>> +
>> +       RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)"
>> +                       " failed", pci_drv->driver.name,
>> +                       (unsigned int) pci_dev->id.vendor_id,
>> +                       (unsigned int) pci_dev->id.device_id);
>> +
>> +       rte_event_pmd_release(eventdev);
>> +
>> +       return -ENXIO;
>> +}
>>
>>   /**
>>    * @internal
>> --
>> 2.13.6
>>

Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

[McDaniel, Timothy]

Hello Jerin,

I am working on V2 of the patchset, and the ABI breakage will be corrected in that version.

Thanks,
Tim

-----Original Message-----
From: Jerin Jacob <jerinjacobk@gmail.com> 
Sent: Friday, June 12, 2020 10:59 PM
To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
Cc: Jerin Jacob <jerinj@marvell.com>; dpdk-dev <dev@dpdk.org>; Eads, Gage <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>; Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Subject: Re: [dpdk-dev] [PATCH 01/27] eventdev: dlb upstream prerequisites

On Sat, Jun 13, 2020 at 2:56 AM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
>
> The DLB hardware does not conform exactly to the eventdev interface.
> 1) It has a limit on the number of queues that may be linked to a port.
> 2) Some ports a further restricted to a maximum of 1 linked queue.
> 3) It does not (currently) have the ability to carry the flow_id as part
> of the event (QE) payload.
>
> Due to the above, we would like to propose the following enhancements.


Thanks, McDaniel, Good to see new HW PMD for eventdev.

+ Ray and Neil.

Hello McDaniel,
I assume this patchset is for v20.08. It is adding new elements in
pubic structures. Have you checked the ABI breakage?

I will review the rest of the series if there is NO ABI breakage as we
can not have the ABI breakage 20.08 version.


ABI validator
~~~~~~~~~~~~~~
1. meson build
2.  Compile and install known stable abi libs i.e ToT.
         DESTDIR=$PWD/install-meson-stable ninja -C build install
     Compile and install with patches to be verified.
         DESTDIR=$PWD/install-meson-new ninja -C build install
3. Gen ABI for both
        devtools/gen-abi.sh install-meson-stable
        devtools/gen-abi.sh install-meson-new
4. Run abi checker
        devtools/check-abi.sh install-meson-stable install-meson-new


DPDK_ABI_REF_DIR=/build/dpdk/reference/ DPDK_ABI_REF_VERSION=v20.02
./devtools/test-meson-builds.sh
DPDK_ABI_REF_DIR - needs an absolute path, for reasons that are still
unclear to me.
DPDK_ABI_REF_VERSION - you need to use the last DPDK release.

>
> 1) Add new fields to the rte_event_dev_info struct. These fields allow
> the device to advertize its capabilities so that applications can take
> the appropriate actions based on those capabilities.
>
>     struct rte_event_dev_info {
>         uint32_t max_event_port_links;
>         /**< Maximum number of queues that can be linked to a single event
>          * port by this device.
>          */
>
>         uint8_t max_single_link_event_port_queue_pairs;
>         /**< Maximum number of event ports and queues that are optimized for
>          * (and only capable of) single-link configurations supported by this
>          * device. These ports and queues are not accounted for in
>          * max_event_ports or max_event_queues.
>          */
>     }
>
> 2) Add a new field to the rte_event_dev_config struct. This field allows the
> application to specify how many of its ports are limited to a single link,
> or will be used in single link mode.
>
>     /** Event device configuration structure */
>     struct rte_event_dev_config {
>         uint8_t nb_single_link_event_port_queues;
>         /**< Number of event ports and queues that will be singly-linked to
>          * each other. These are a subset of the overall event ports and
>          * queues; this value cannot exceed *nb_event_ports* or
>          * *nb_event_queues*. If the device has ports and queues that are
>          * optimized for single-link usage, this field is a hint for how many
>          * to allocate; otherwise, regular event ports and queues can be used.
>          */
>     }
>
> 3) Replace the dedicated implicit_release_disabled field with a bit field
> of explicit port capabilities. The implicit_release_disable functionality
> is assiged to one bit, and a port-is-single-link-only  attribute is
> assigned to other, with the remaining bits available for future assignment.
>
>         * Event port configuration bitmap flags */
>         #define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
>         /**< Configure the port not to release outstanding events in
>          * rte_event_dev_dequeue_burst(). If set, all events received through
>          * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
>          * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
>          * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
>          */
>         #define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
>
>         /**< This event port links only to a single event queue.
>          *
>          *  @see rte_event_port_setup(), rte_event_port_link()
>          */
>
>         #define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>         /**
>          * The implicit release disable attribute of the port
>          */
>
>         struct rte_event_port_conf {
>                 uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>         }
>
> 4) Add UMWAIT/UMONITOR bit to rte_cpuflags
>
> 5) Added a new API that is useful for probing PCI devices.
>
>         /**
>          * @internal
>          * Wrapper for use by pci drivers as a .probe function to attach to a event
>          * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
>          * the name.
>          */
>         static inline int
>         rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
>                                     struct rte_pci_device *pci_dev,
>                                     size_t private_data_size,
>                                     eventdev_pmd_pci_callback_t devinit,
>                                     const char *name);
>
> Change-Id: I4cf00015296e2b3feca9886895765554730594be
> Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> ---
>  app/test-eventdev/evt_common.h                     |  1 +
>  app/test-eventdev/test_order_atq.c                 |  4 ++
>  app/test-eventdev/test_order_common.c              |  6 ++-
>  app/test-eventdev/test_order_queue.c               |  4 ++
>  app/test-eventdev/test_perf_atq.c                  |  1 +
>  app/test-eventdev/test_perf_queue.c                |  1 +
>  app/test-eventdev/test_pipeline_atq.c              |  1 +
>  app/test-eventdev/test_pipeline_queue.c            |  1 +
>  app/test/test_eventdev.c                           |  4 +-
>  drivers/event/dpaa2/dpaa2_eventdev.c               |  2 +-
>  drivers/event/octeontx/ssovf_evdev.c               |  2 +-
>  drivers/event/skeleton/skeleton_eventdev.c         |  2 +-
>  drivers/event/sw/sw_evdev.c                        |  5 +-
>  drivers/event/sw/sw_evdev_selftest.c               |  9 ++--
>  .../eventdev_pipeline/pipeline_worker_generic.c    |  8 ++-
>  examples/eventdev_pipeline/pipeline_worker_tx.c    |  3 ++
>  examples/l2fwd-event/l2fwd_event_generic.c         |  5 +-
>  examples/l2fwd-event/l2fwd_event_internal_port.c   |  5 +-
>  examples/l3fwd/l3fwd_event_generic.c               |  5 +-
>  examples/l3fwd/l3fwd_event_internal_port.c         |  5 +-
>  lib/librte_eal/x86/include/rte_cpuflags.h          |  1 +
>  lib/librte_eal/x86/rte_cpuflags.c                  |  1 +
>  lib/librte_eventdev/rte_event_eth_tx_adapter.c     |  2 +-
>  lib/librte_eventdev/rte_eventdev.c                 | 62 +++++++++++++++++++---
>  lib/librte_eventdev/rte_eventdev.h                 | 51 +++++++++++++++---
>  lib/librte_eventdev/rte_eventdev_pmd_pci.h         | 54 +++++++++++++++++++
>  26 files changed, 208 insertions(+), 37 deletions(-)
>
> diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
> index f9d7378d3..120c27b33 100644
> --- a/app/test-eventdev/evt_common.h
> +++ b/app/test-eventdev/evt_common.h
> @@ -169,6 +169,7 @@ evt_configure_eventdev(struct evt_options *opt, uint8_t nb_queues,
>                         .dequeue_timeout_ns = opt->deq_tmo_nsec,
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 0,
>                         .nb_events_limit  = info.max_num_events,
>                         .nb_event_queue_flows = opt->nb_flows,
>                         .nb_event_port_dequeue_depth =
> diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
> index 3366cfce9..8246b96f0 100644
> --- a/app/test-eventdev/test_order_atq.c
> +++ b/app/test-eventdev/test_order_atq.c
> @@ -34,6 +34,8 @@ order_atq_worker(void *arg)
>                         continue;
>                 }
>
> +               ev.flow_id = ev.mbuf->udata64;
> +
>                 if (ev.sub_event_type == 0) { /* stage 0 from producer */
>                         order_atq_process_stage_0(&ev);
>                         while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
> @@ -68,6 +70,8 @@ order_atq_worker_burst(void *arg)
>                 }
>
>                 for (i = 0; i < nb_rx; i++) {
> +                       ev[i].flow_id = ev[i].mbuf->udata64;
> +
>                         if (ev[i].sub_event_type == 0) { /*stage 0 */
>                                 order_atq_process_stage_0(&ev[i]);
>                         } else if (ev[i].sub_event_type == 1) { /* stage 1 */
> diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
> index 4190f9ade..c6fcd0509 100644
> --- a/app/test-eventdev/test_order_common.c
> +++ b/app/test-eventdev/test_order_common.c
> @@ -49,6 +49,7 @@ order_producer(void *arg)
>                 const uint32_t flow = (uintptr_t)m % nb_flows;
>                 /* Maintain seq number per flow */
>                 m->seqn = producer_flow_seq[flow]++;
> +               m->udata64 = flow;
>
>                 ev.flow_id = flow;
>                 ev.mbuf = m;
> @@ -318,10 +319,11 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>                 opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
>
>         /* port configuration */
> -       const struct rte_event_port_conf p_conf = {
> +       struct rte_event_port_conf p_conf = {
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         /* setup one port per worker, linking to all queues */
> @@ -351,6 +353,8 @@ order_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
>         p->queue_id = 0;
>         p->t = t;
>
> +       p_conf.new_event_threshold /= 2;
> +
>         ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
>         if (ret) {
>                 evt_err("failed to setup producer port %d", port);
> diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
> index 495efd92f..a0a2187a2 100644
> --- a/app/test-eventdev/test_order_queue.c
> +++ b/app/test-eventdev/test_order_queue.c
> @@ -34,6 +34,8 @@ order_queue_worker(void *arg)
>                         continue;
>                 }
>
> +               ev.flow_id = ev.mbuf->udata64;
> +
>                 if (ev.queue_id == 0) { /* from ordered queue */
>                         order_queue_process_stage_0(&ev);
>                         while (rte_event_enqueue_burst(dev_id, port, &ev, 1)
> @@ -68,6 +70,8 @@ order_queue_worker_burst(void *arg)
>                 }
>
>                 for (i = 0; i < nb_rx; i++) {
> +                       ev[i].flow_id = ev[i].mbuf->udata64;
> +
>                         if (ev[i].queue_id == 0) { /* from ordered queue */
>                                 order_queue_process_stage_0(&ev[i]);
>                         } else if (ev[i].queue_id == 1) {/* from atomic queue */
> diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
> index 8fd51004e..10846f202 100644
> --- a/app/test-eventdev/test_perf_atq.c
> +++ b/app/test-eventdev/test_perf_atq.c
> @@ -204,6 +204,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues,
> diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
> index f4ea3a795..a0119da60 100644
> --- a/app/test-eventdev/test_perf_queue.c
> +++ b/app/test-eventdev/test_perf_queue.c
> @@ -219,6 +219,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = dev_info.max_event_port_dequeue_depth,
>                         .new_event_threshold = dev_info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */,
> diff --git a/app/test-eventdev/test_pipeline_atq.c b/app/test-eventdev/test_pipeline_atq.c
> index 8e8686c14..a95ec0aa5 100644
> --- a/app/test-eventdev/test_pipeline_atq.c
> +++ b/app/test-eventdev/test_pipeline_atq.c
> @@ -356,6 +356,7 @@ pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                 .dequeue_depth = opt->wkr_deq_dep,
>                 .enqueue_depth = info.max_event_port_dequeue_depth,
>                 .new_event_threshold = info.max_num_events,
> +               .event_port_cfg = 0,
>         };
>
>         if (!t->internal_port)
> diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c
> index 7bebac34f..30817dc78 100644
> --- a/app/test-eventdev/test_pipeline_queue.c
> +++ b/app/test-eventdev/test_pipeline_queue.c
> @@ -379,6 +379,7 @@ pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
>                         .dequeue_depth = opt->wkr_deq_dep,
>                         .enqueue_depth = info.max_event_port_dequeue_depth,
>                         .new_event_threshold = info.max_num_events,
> +                       .event_port_cfg = 0,
>         };
>
>         if (!t->internal_port) {
> diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
> index 43ccb1ce9..62019c185 100644
> --- a/app/test/test_eventdev.c
> +++ b/app/test/test_eventdev.c
> @@ -559,10 +559,10 @@ test_eventdev_port_setup(void)
>         if (!(info.event_dev_cap &
>               RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>                 pconf.enqueue_depth = info.max_event_port_enqueue_depth;
> -               pconf.disable_implicit_release = 1;
> +               pconf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>                 ret = rte_event_port_setup(TEST_DEV_ID, 0, &pconf);
>                 TEST_ASSERT(ret == -EINVAL, "Expected -EINVAL, %d", ret);
> -               pconf.disable_implicit_release = 0;
> +               pconf.event_port_cfg = 0;
>         }
>
>         ret = rte_event_port_setup(TEST_DEV_ID, info.max_event_ports,
> diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c
> index a196ad4c6..8568bfcfc 100644
> --- a/drivers/event/dpaa2/dpaa2_eventdev.c
> +++ b/drivers/event/dpaa2/dpaa2_eventdev.c
> @@ -537,7 +537,7 @@ dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>                 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
>         port_conf->enqueue_depth =
>                 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static int
> diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c
> index 1b1a5d939..99c0b2efb 100644
> --- a/drivers/event/octeontx/ssovf_evdev.c
> +++ b/drivers/event/octeontx/ssovf_evdev.c
> @@ -224,7 +224,7 @@ ssovf_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = edev->max_num_events;
>         port_conf->dequeue_depth = 1;
>         port_conf->enqueue_depth = 1;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static void
> diff --git a/drivers/event/skeleton/skeleton_eventdev.c b/drivers/event/skeleton/skeleton_eventdev.c
> index c889220e0..37d569b8c 100644
> --- a/drivers/event/skeleton/skeleton_eventdev.c
> +++ b/drivers/event/skeleton/skeleton_eventdev.c
> @@ -209,7 +209,7 @@ skeleton_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = 32 * 1024;
>         port_conf->dequeue_depth = 16;
>         port_conf->enqueue_depth = 16;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static void
> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
> index fb8e8bebb..0b3dd9c1c 100644
> --- a/drivers/event/sw/sw_evdev.c
> +++ b/drivers/event/sw/sw_evdev.c
> @@ -175,7 +175,8 @@ sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
>         }
>
>         p->inflight_max = conf->new_event_threshold;
> -       p->implicit_release = !conf->disable_implicit_release;
> +       p->implicit_release = !(conf->event_port_cfg &
> +                               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
>
>         /* check if ring exists, same as rx_worker above */
>         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
> @@ -508,7 +509,7 @@ sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
>         port_conf->new_event_threshold = 1024;
>         port_conf->dequeue_depth = 16;
>         port_conf->enqueue_depth = 16;
> -       port_conf->disable_implicit_release = 0;
> +       port_conf->event_port_cfg = 0;
>  }
>
>  static int
> diff --git a/drivers/event/sw/sw_evdev_selftest.c b/drivers/event/sw/sw_evdev_selftest.c
> index 38c21fa0f..a78d6cd0d 100644
> --- a/drivers/event/sw/sw_evdev_selftest.c
> +++ b/drivers/event/sw/sw_evdev_selftest.c
> @@ -172,7 +172,7 @@ create_ports(struct test *t, int num_ports)
>                         .new_event_threshold = 1024,
>                         .dequeue_depth = 32,
>                         .enqueue_depth = 64,
> -                       .disable_implicit_release = 0,
> +                       .event_port_cfg = 0,
>         };
>         if (num_ports > MAX_PORTS)
>                 return -1;
> @@ -1227,7 +1227,7 @@ port_reconfig_credits(struct test *t)
>                                 .new_event_threshold = 128,
>                                 .dequeue_depth = 32,
>                                 .enqueue_depth = 64,
> -                               .disable_implicit_release = 0,
> +                               .event_port_cfg = 0,
>                 };
>                 if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>                         printf("%d Error setting up port\n", __LINE__);
> @@ -1317,7 +1317,7 @@ port_single_lb_reconfig(struct test *t)
>                 .new_event_threshold = 128,
>                 .dequeue_depth = 32,
>                 .enqueue_depth = 64,
> -               .disable_implicit_release = 0,
> +               .event_port_cfg = 0,
>         };
>         if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
>                 printf("%d Error setting up port\n", __LINE__);
> @@ -3079,7 +3079,8 @@ worker_loopback(struct test *t, uint8_t disable_implicit_release)
>          * only be initialized once - and this needs to be set for multiple runs
>          */
>         conf.new_event_threshold = 512;
> -       conf.disable_implicit_release = disable_implicit_release;
> +       conf.event_port_cfg = disable_implicit_release ?
> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>
>         if (rte_event_port_setup(evdev, 0, &conf) < 0) {
>                 printf("Error setting up RX port\n");
> diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
> index 42ff4eeb9..a091da3ba 100644
> --- a/examples/eventdev_pipeline/pipeline_worker_generic.c
> +++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
> @@ -129,6 +129,7 @@ setup_eventdev_generic(struct worker_data *worker_data)
>         struct rte_event_dev_config config = {
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 1,
>                         .nb_events_limit  = 4096,
>                         .nb_event_queue_flows = 1024,
>                         .nb_event_port_dequeue_depth = 128,
> @@ -138,12 +139,13 @@ setup_eventdev_generic(struct worker_data *worker_data)
>                         .dequeue_depth = cdata.worker_cq_depth,
>                         .enqueue_depth = 64,
>                         .new_event_threshold = 4096,
> +                       .event_port_cfg = 0,
>         };
>         struct rte_event_queue_conf wkr_q_conf = {
>                         .schedule_type = cdata.queue_type,
>                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
>                         .nb_atomic_flows = 1024,
> -               .nb_atomic_order_sequences = 1024,
> +                       .nb_atomic_order_sequences = 1024,
>         };
>         struct rte_event_queue_conf tx_q_conf = {
>                         .priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
> @@ -167,7 +169,8 @@ setup_eventdev_generic(struct worker_data *worker_data)
>         disable_implicit_release = (dev_info.event_dev_cap &
>                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
>
> -       wkr_p_conf.disable_implicit_release = disable_implicit_release;
> +       wkr_p_conf.event_port_cfg = disable_implicit_release ?
> +               RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL : 0;
>
>         if (dev_info.max_num_events < config.nb_events_limit)
>                 config.nb_events_limit = dev_info.max_num_events;
> @@ -417,6 +420,7 @@ init_adapters(uint16_t nb_ports)
>                 .dequeue_depth = cdata.worker_cq_depth,
>                 .enqueue_depth = 64,
>                 .new_event_threshold = 4096,
> +               .event_port_cfg = 0,
>         };
>
>         if (adptr_p_conf.new_event_threshold > dev_info.max_num_events)
> diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
> index 55bb2f762..e8a9652aa 100644
> --- a/examples/eventdev_pipeline/pipeline_worker_tx.c
> +++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
> @@ -436,6 +436,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>         struct rte_event_dev_config config = {
>                         .nb_event_queues = nb_queues,
>                         .nb_event_ports = nb_ports,
> +                       .nb_single_link_event_port_queues = 0,
>                         .nb_events_limit  = 4096,
>                         .nb_event_queue_flows = 1024,
>                         .nb_event_port_dequeue_depth = 128,
> @@ -445,6 +446,7 @@ setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
>                         .dequeue_depth = cdata.worker_cq_depth,
>                         .enqueue_depth = 64,
>                         .new_event_threshold = 4096,
> +                       .event_port_cfg = 0,
>         };
>         struct rte_event_queue_conf wkr_q_conf = {
>                         .schedule_type = cdata.queue_type,
> @@ -746,6 +748,7 @@ init_adapters(uint16_t nb_ports)
>                 .dequeue_depth = cdata.worker_cq_depth,
>                 .enqueue_depth = 64,
>                 .new_event_threshold = 4096,
> +               .event_port_cfg = 0,
>         };
>
>         init_ports(nb_ports);
> diff --git a/examples/l2fwd-event/l2fwd_event_generic.c b/examples/l2fwd-event/l2fwd_event_generic.c
> index 2dc95e5f7..e01df0435 100644
> --- a/examples/l2fwd-event/l2fwd_event_generic.c
> +++ b/examples/l2fwd-event/l2fwd_event_generic.c
> @@ -126,8 +126,9 @@ l2fwd_event_port_setup_generic(struct l2fwd_resources *rsrc)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>         evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
> diff --git a/examples/l2fwd-event/l2fwd_event_internal_port.c b/examples/l2fwd-event/l2fwd_event_internal_port.c
> index 63d57b46c..f54327b4f 100644
> --- a/examples/l2fwd-event/l2fwd_event_internal_port.c
> +++ b/examples/l2fwd-event/l2fwd_event_internal_port.c
> @@ -123,8 +123,9 @@ l2fwd_event_port_setup_internal_port(struct l2fwd_resources *rsrc)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>                                                                 event_p_id++) {
> diff --git a/examples/l3fwd/l3fwd_event_generic.c b/examples/l3fwd/l3fwd_event_generic.c
> index f8c98435d..409a4107e 100644
> --- a/examples/l3fwd/l3fwd_event_generic.c
> +++ b/examples/l3fwd/l3fwd_event_generic.c
> @@ -115,8 +115,9 @@ l3fwd_event_port_setup_generic(void)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>         evt_rsrc->deq_depth = def_p_conf.dequeue_depth;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
> diff --git a/examples/l3fwd/l3fwd_event_internal_port.c b/examples/l3fwd/l3fwd_event_internal_port.c
> index 03ac581d6..df410f10f 100644
> --- a/examples/l3fwd/l3fwd_event_internal_port.c
> +++ b/examples/l3fwd/l3fwd_event_internal_port.c
> @@ -113,8 +113,9 @@ l3fwd_event_port_setup_internal_port(void)
>         if (def_p_conf.enqueue_depth < event_p_conf.enqueue_depth)
>                 event_p_conf.enqueue_depth = def_p_conf.enqueue_depth;
>
> -       event_p_conf.disable_implicit_release =
> -               evt_rsrc->disable_implicit_release;
> +       event_p_conf.event_port_cfg = 0;
> +       if (evt_rsrc->disable_implicit_release)
> +               event_p_conf.event_port_cfg |= RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
>
>         for (event_p_id = 0; event_p_id < evt_rsrc->evp.nb_ports;
>                                                                 event_p_id++) {
> diff --git a/lib/librte_eal/x86/include/rte_cpuflags.h b/lib/librte_eal/x86/include/rte_cpuflags.h
> index c1d20364d..ab2c3b379 100644
> --- a/lib/librte_eal/x86/include/rte_cpuflags.h
> +++ b/lib/librte_eal/x86/include/rte_cpuflags.h
> @@ -130,6 +130,7 @@ enum rte_cpu_flag_t {
>         RTE_CPUFLAG_CLDEMOTE,               /**< Cache Line Demote */
>         RTE_CPUFLAG_MOVDIRI,                /**< Direct Store Instructions */
>         RTE_CPUFLAG_MOVDIR64B,              /**< Direct Store Instructions 64B */
> +       RTE_CPUFLAG_UMWAIT,                 /**< UMONITOR/UMWAIT */
>         RTE_CPUFLAG_AVX512VP2INTERSECT,     /**< AVX512 Two Register Intersection */
>
>         /* The last item */
> diff --git a/lib/librte_eal/x86/rte_cpuflags.c b/lib/librte_eal/x86/rte_cpuflags.c
> index 30439e795..69ac0dbce 100644
> --- a/lib/librte_eal/x86/rte_cpuflags.c
> +++ b/lib/librte_eal/x86/rte_cpuflags.c
> @@ -137,6 +137,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
>         FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25)
>         FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27)
>         FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28)
> +        FEAT_DEF(UMWAIT, 0x00000007, 0, RTE_REG_ECX, 5)
>         FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8)
>  };
>
> diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> index bb21dc407..8a72256de 100644
> --- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> +++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
> @@ -286,7 +286,7 @@ txa_service_conf_cb(uint8_t __rte_unused id, uint8_t dev_id,
>                 return ret;
>         }
>
> -       pc->disable_implicit_release = 0;
> +       pc->event_port_cfg = 0;
>         ret = rte_event_port_setup(dev_id, port_id, pc);
>         if (ret) {
>                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n",
> diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
> index 82c177c73..4955ab1a0 100644
> --- a/lib/librte_eventdev/rte_eventdev.c
> +++ b/lib/librte_eventdev/rte_eventdev.c
> @@ -437,9 +437,29 @@ rte_event_dev_configure(uint8_t dev_id,
>                                         dev_id);
>                 return -EINVAL;
>         }
> -       if (dev_conf->nb_event_queues > info.max_event_queues) {
> -               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d",
> -               dev_id, dev_conf->nb_event_queues, info.max_event_queues);
> +       if (dev_conf->nb_event_queues > info.max_event_queues +
> +                       info.max_single_link_event_port_queue_pairs) {
> +               RTE_EDEV_LOG_ERR("%d nb_event_queues=%d > max_event_queues=%d + max_single_link_event_port_queue_pairs=%d",
> +                                dev_id, dev_conf->nb_event_queues,
> +                                info.max_event_queues,
> +                                info.max_single_link_event_port_queue_pairs);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_event_queues -
> +                       dev_conf->nb_single_link_event_port_queues >
> +                       info.max_event_queues) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_queues=%d - nb_single_link_event_port_queues=%d > max_event_queues=%d",
> +                                dev_id, dev_conf->nb_event_queues,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                info.max_event_queues);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_single_link_event_port_queues >
> +                       dev_conf->nb_event_queues) {
> +               RTE_EDEV_LOG_ERR("dev%d nb_single_link_event_port_queues=%d > nb_event_queues=%d",
> +                                dev_id,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                dev_conf->nb_event_queues);
>                 return -EINVAL;
>         }
>
> @@ -448,9 +468,31 @@ rte_event_dev_configure(uint8_t dev_id,
>                 RTE_EDEV_LOG_ERR("dev%d nb_event_ports cannot be zero", dev_id);
>                 return -EINVAL;
>         }
> -       if (dev_conf->nb_event_ports > info.max_event_ports) {
> -               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports= %d",
> -               dev_id, dev_conf->nb_event_ports, info.max_event_ports);
> +       if (dev_conf->nb_event_ports > info.max_event_ports +
> +                       info.max_single_link_event_port_queue_pairs) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d > max_event_ports=%d + max_single_link_event_port_queue_pairs=%d",
> +                                dev_id, dev_conf->nb_event_ports,
> +                                info.max_event_ports,
> +                                info.max_single_link_event_port_queue_pairs);
> +               return -EINVAL;
> +       }
> +       if (dev_conf->nb_event_ports -
> +                       dev_conf->nb_single_link_event_port_queues
> +                       > info.max_event_ports) {
> +               RTE_EDEV_LOG_ERR("id%d nb_event_ports=%d - nb_single_link_event_port_queues=%d > max_event_ports=%d",
> +                                dev_id, dev_conf->nb_event_ports,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                info.max_event_ports);
> +               return -EINVAL;
> +       }
> +
> +       if (dev_conf->nb_single_link_event_port_queues >
> +           dev_conf->nb_event_ports) {
> +               RTE_EDEV_LOG_ERR(
> +                                "dev%d nb_single_link_event_port_queues=%d > nb_event_ports=%d",
> +                                dev_id,
> +                                dev_conf->nb_single_link_event_port_queues,
> +                                dev_conf->nb_event_ports);
>                 return -EINVAL;
>         }
>
> @@ -737,7 +779,8 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>                 return -EINVAL;
>         }
>
> -       if (port_conf && port_conf->disable_implicit_release &&
> +       if (port_conf &&
> +           (port_conf->event_port_cfg & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL) &&
>             !(dev->data->event_dev_cap &
>               RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE)) {
>                 RTE_EDEV_LOG_ERR(
> @@ -809,6 +852,7 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>                         uint32_t *attr_value)
>  {
>         struct rte_eventdev *dev;
> +       uint32_t config;
>
>         if (!attr_value)
>                 return -EINVAL;
> @@ -830,6 +874,10 @@ rte_event_port_attr_get(uint8_t dev_id, uint8_t port_id, uint32_t attr_id,
>         case RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD:
>                 *attr_value = dev->data->ports_cfg[port_id].new_event_threshold;
>                 break;
> +       case RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE:
> +               config = dev->data->ports_cfg[port_id].event_port_cfg;
> +               *attr_value = !!(config & RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
> +               break;
>         default:
>                 return -EINVAL;
>         };
> diff --git a/lib/librte_eventdev/rte_eventdev.h b/lib/librte_eventdev/rte_eventdev.h
> index 7dc832353..7f7a8a275 100644
> --- a/lib/librte_eventdev/rte_eventdev.h
> +++ b/lib/librte_eventdev/rte_eventdev.h
> @@ -291,6 +291,13 @@ struct rte_event;
>   * single queue to each port or map a single queue to many port.
>   */
>
> +#define RTE_EVENT_DEV_CAP_CARRY_FLOW_ID (1ULL << 9)
> +/**< Event device is capable of carrying the flow ID from the enqueued
> + * event to the dequeued event. If the flag is set, the dequeued event's flow
> + * ID matches the corresponding enqueued event's flow ID. If the flag is not
> + * set, the dequeued event's flow ID field is uninitialized.
> + */
> +
>  /* Event device priority levels */
>  #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
>  /**< Highest priority expressed across eventdev subsystem
> @@ -380,6 +387,10 @@ struct rte_event_dev_info {
>          * event port by this device.
>          * A device that does not support bulk enqueue will set this as 1.
>          */
> +       uint32_t max_event_port_links;
> +       /**< Maximum number of queues that can be linked to a single event
> +        * port by this device.
> +        */
>         int32_t max_num_events;
>         /**< A *closed system* event dev has a limit on the number of events it
>          * can manage at a time. An *open system* event dev does not have a
> @@ -387,6 +398,12 @@ struct rte_event_dev_info {
>          */
>         uint32_t event_dev_cap;
>         /**< Event device capabilities(RTE_EVENT_DEV_CAP_)*/
> +       uint8_t max_single_link_event_port_queue_pairs;
> +       /**< Maximum number of event ports and queues that are optimized for
> +        * (and only capable of) single-link configurations supported by this
> +        * device. These ports and queues are not accounted for in
> +        * max_event_ports or max_event_queues.
> +        */
>  };
>
>  /**
> @@ -494,6 +511,14 @@ struct rte_event_dev_config {
>          */
>         uint32_t event_dev_cfg;
>         /**< Event device config flags(RTE_EVENT_DEV_CFG_)*/
> +       uint8_t nb_single_link_event_port_queues;
> +       /**< Number of event ports and queues that will be singly-linked to
> +        * each other. These are a subset of the overall event ports and
> +        * queues; this value cannot exceed *nb_event_ports* or
> +        * *nb_event_queues*. If the device has ports and queues that are
> +        * optimized for single-link usage, this field is a hint for how many
> +        * to allocate; otherwise, regular event ports and queues can be used.
> +        */
>  };
>
>  /**
> @@ -671,6 +696,20 @@ rte_event_queue_attr_get(uint8_t dev_id, uint8_t queue_id, uint32_t attr_id,
>
>  /* Event port specific APIs */
>
> +/* Event port configuration bitmap flags */
> +#define RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL    (1ULL << 0)
> +/**< Configure the port not to release outstanding events in
> + * rte_event_dev_dequeue_burst(). If set, all events received through
> + * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
> + * RTE_EVENT_OP_FORWARD. Must be unset if the device is not
> + * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
> + */
> +#define RTE_EVENT_PORT_CFG_SINGLE_LINK         (1ULL << 1)
> +/**< This event port links only to a single event queue.
> + *
> + *  @see rte_event_port_setup(), rte_event_port_link()
> + */
> +
>  /** Event port configuration structure */
>  struct rte_event_port_conf {
>         int32_t new_event_threshold;
> @@ -698,13 +737,7 @@ struct rte_event_port_conf {
>          * which previously supplied to rte_event_dev_configure().
>          * Ignored when device is not RTE_EVENT_DEV_CAP_BURST_MODE capable.
>          */
> -       uint8_t disable_implicit_release;
> -       /**< Configure the port not to release outstanding events in
> -        * rte_event_dev_dequeue_burst(). If true, all events received through
> -        * the port must be explicitly released with RTE_EVENT_OP_RELEASE or
> -        * RTE_EVENT_OP_FORWARD. Must be false when the device is not
> -        * RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE capable.
> -        */
> +       uint32_t event_port_cfg; /**< Port cfg flags(EVENT_PORT_CFG_) */
>  };
>
>  /**
> @@ -769,6 +802,10 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
>   * The new event threshold of the port
>   */
>  #define RTE_EVENT_PORT_ATTR_NEW_EVENT_THRESHOLD 2
> +/**
> + * The implicit release disable attribute of the port
> + */
> +#define RTE_EVENT_PORT_ATTR_IMPLICIT_RELEASE_DISABLE 3
>
>  /**
>   * Get an attribute from a port.
> diff --git a/lib/librte_eventdev/rte_eventdev_pmd_pci.h b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> index 443cd38c2..157299983 100644
> --- a/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> +++ b/lib/librte_eventdev/rte_eventdev_pmd_pci.h
> @@ -88,6 +88,60 @@ rte_event_pmd_pci_probe(struct rte_pci_driver *pci_drv,
>         return -ENXIO;
>  }
>
> +/**
> + * @internal
> + * Wrapper for use by pci drivers as a .probe function to attach to a event
> + * interface.  Same as rte_event_pmd_pci_probe, except caller can specify
> + * the name.
> + */
> +static inline int
> +rte_event_pmd_pci_probe_named(struct rte_pci_driver *pci_drv,
> +                           struct rte_pci_device *pci_dev,
> +                           size_t private_data_size,
> +                           eventdev_pmd_pci_callback_t devinit,
> +                           const char *name)
> +{
> +       struct rte_eventdev *eventdev;
> +
> +       int retval;
> +
> +       if (devinit == NULL)
> +               return -EINVAL;
> +
> +       eventdev = rte_event_pmd_allocate(name,
> +                        pci_dev->device.numa_node);
> +       if (eventdev == NULL)
> +               return -ENOMEM;
> +
> +       if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> +               eventdev->data->dev_private =
> +                               rte_zmalloc_socket(
> +                                               "eventdev private structure",
> +                                               private_data_size,
> +                                               RTE_CACHE_LINE_SIZE,
> +                                               rte_socket_id());
> +
> +               if (eventdev->data->dev_private == NULL)
> +                       rte_panic("Cannot allocate memzone for private "
> +                                       "device data");
> +       }
> +
> +       eventdev->dev = &pci_dev->device;
> +
> +       /* Invoke PMD device initialization function */
> +       retval = devinit(eventdev);
> +       if (retval == 0)
> +               return 0;
> +
> +       RTE_EDEV_LOG_ERR("driver %s: (vendor_id=0x%x device_id=0x%x)"
> +                       " failed", pci_drv->driver.name,
> +                       (unsigned int) pci_dev->id.vendor_id,
> +                       (unsigned int) pci_dev->id.device_id);
> +
> +       rte_event_pmd_release(eventdev);
> +
> +       return -ENXIO;
> +}
>
>  /**
>   * @internal
> --
> 2.13.6
>

[dpdk-dev] [PATCH v7 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
Depends-on: patch-79539 ("eal: add new x86 cpuid support for WAITPKG")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4129 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1551 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21764 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v7 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Increased RTE_EVENT_MAX_QUEUES_PER_DEV config option
from 64 to 255.

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                    |  6 +++++-
 config/rte_config.h            |  6 ++++++
 doc/guides/eventdevs/dlb.rst   | 36 ++++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst |  1 +
 drivers/event/dlb/meson.build  | 13 +++++++++++++
 drivers/event/dlb/version.map  |  3 +++
 drivers/event/meson.build      |  2 +-
 7 files changed, 65 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v7 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v7 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f695abf
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+}; __rte_cache_aligned
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v7 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v7 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..7bbe69e
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__v2di src_data0 = (__v2di){_src[0], _src[1]};
+
+	__builtin_ia32_movntdq((__v2di *)dest, (__v2di)src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v7 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f695abf..892d55f 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v7 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v7 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v7 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v7 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..3ac7393 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v7 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v7 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 3ac7393..4557ee5 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v7 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanded (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 4557ee5..f5fb055 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v7 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v7 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v7 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v7 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 161 ++++++++++
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 843 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f5fb055..12142b1 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v7 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 728 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 749 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 12142b1..6940ef5 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..c022139 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2812,9 +2812,728 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline bool
+dlb_cq_is_empty(struct dlb_port *qm_port)
+{
+	volatile struct dlb_dequeue_qe *qe_ptr;
+	struct dlb_dequeue_qe qe;
+
+	qe_ptr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	qe = qe_ptr[qm_port->cq_idx];
+
+	return (qe.cq_gen != qm_port->gen_bit);
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+
+		cq_base = port_data->cq_base;
+
+		/* Block on cache line write to CQ. Note: it's
+		 * safe to access the per-process cq_base
+		 * address here, since the PMD has already
+		 * attempted at least one CQ dequeue.
+		 */
+		dlb_umonitor(&cq_base[qm_port->cq_idx]);
+
+		/* Avoid race condition. Check if still empty */
+		if (dlb_cq_is_empty(qm_port)) {
+			dlb_umwait(RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE,
+				   timeout + start_ticks);
+			DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait,
+				     1);
+		}
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_qe(struct dlb_eventdev_port *ev_port __rte_unused,
+		       struct dlb_port *qm_port,
+		       struct rte_event *event,
+		       struct dlb_dequeue_qe *qe)
+{
+	int sched_type_map[4] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+
+	DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+		    (long long)qe->data, qe->qid,
+		    qe->u.event_type.major,
+		    qe->u.event_type.sub,
+		    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+	/* Fill in event information.
+	 * Note that flow_id must be embedded in the data by
+	 * the app, such as the mbuf RSS hash field if the data
+	 * buffer is a mbuf.
+	 */
+	if (unlikely(qe->error)) {
+		DLB_LOG_ERR("QE error bit ON\n");
+		DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+		dlb_consume_qe_immediate(qm_port, 1);
+		return 0;
+	}
+
+	event->u64 = qe->data;
+	event->queue_id = qid_mappings[qe->qid];
+	event->priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+	event->event_type = qe->u.event_type.major;
+	event->sub_event_type = qe->u.event_type.sub;
+	event->sched_type = sched_type_map[qe->sched_type];
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, 1);
+
+	return 1;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3560,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v7 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c022139..cdabc9b 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -90,17 +90,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3529,6 +3518,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3538,6 +3770,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v7 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index cdabc9b..4e1af0a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1021,6 +1021,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1181,6 +1208,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2681,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2709,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2744,7 +2803,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2752,21 +2811,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2774,7 +2833,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2782,7 +2841,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2790,7 +2849,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2798,7 +2857,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3203,7 +3262,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3373,7 +3433,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3417,6 +3478,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3449,6 +3532,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3468,6 +3552,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3486,6 +3574,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3505,6 +3594,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3811,7 +3904,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3860,6 +3953,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index 892d55f..258d4db 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v7 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1551 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1560 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4e1af0a..0585875 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3878,6 +3878,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..2be5520
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+static inline int
+create_ordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
+}
+
+static inline int
+create_unordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v7 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 0585875..aa22d03 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -158,6 +158,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3854,6 +3857,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3868,7 +3893,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v7 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/dlb.c                | 13 +++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index aa22d03..b21c9b1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3879,6 +3879,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3900,6 +3912,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

Re: [dpdk-dev] [PATCH v7 23/23] event/dlb: add timeout ticks entry point

[Thomas Monjalon]

29/10/2020 15:57, Timothy McDaniel:
> Adds the timeout ticks conversion function.
> 
> Adds announcement of availabililty of the new driver
> for Intel Dynamic Load Balancer 1.0 hardware.

I don't see how these 2 changes are related?

Shouldn't you update the release notes in the first patch?

[dpdk-dev] [PATCH v8 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
Depends-on: patch-79539 ("eal: add new x86 cpuid support for WAITPKG")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4129 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1551 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21764 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v8 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v8 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v8 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f695abf
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+}; __rte_cache_aligned
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v8 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v8 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..7bbe69e
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__v2di src_data0 = (__v2di){_src[0], _src[1]};
+
+	__builtin_ia32_movntdq((__v2di *)dest, (__v2di)src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v8 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f695abf..892d55f 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v8 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v8 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v8 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v8 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v8 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v8 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v8 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanded (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v8 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v8 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v8 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v8 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v8 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 728 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 749 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..c022139 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2812,9 +2812,728 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline bool
+dlb_cq_is_empty(struct dlb_port *qm_port)
+{
+	volatile struct dlb_dequeue_qe *qe_ptr;
+	struct dlb_dequeue_qe qe;
+
+	qe_ptr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	qe = qe_ptr[qm_port->cq_idx];
+
+	return (qe.cq_gen != qm_port->gen_bit);
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+
+		cq_base = port_data->cq_base;
+
+		/* Block on cache line write to CQ. Note: it's
+		 * safe to access the per-process cq_base
+		 * address here, since the PMD has already
+		 * attempted at least one CQ dequeue.
+		 */
+		dlb_umonitor(&cq_base[qm_port->cq_idx]);
+
+		/* Avoid race condition. Check if still empty */
+		if (dlb_cq_is_empty(qm_port)) {
+			dlb_umwait(RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE,
+				   timeout + start_ticks);
+			DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait,
+				     1);
+		}
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_qe(struct dlb_eventdev_port *ev_port __rte_unused,
+		       struct dlb_port *qm_port,
+		       struct rte_event *event,
+		       struct dlb_dequeue_qe *qe)
+{
+	int sched_type_map[4] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+
+	DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+		    (long long)qe->data, qe->qid,
+		    qe->u.event_type.major,
+		    qe->u.event_type.sub,
+		    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+	/* Fill in event information.
+	 * Note that flow_id must be embedded in the data by
+	 * the app, such as the mbuf RSS hash field if the data
+	 * buffer is a mbuf.
+	 */
+	if (unlikely(qe->error)) {
+		DLB_LOG_ERR("QE error bit ON\n");
+		DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+		dlb_consume_qe_immediate(qm_port, 1);
+		return 0;
+	}
+
+	event->u64 = qe->data;
+	event->queue_id = qid_mappings[qe->qid];
+	event->priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+	event->event_type = qe->u.event_type.major;
+	event->sub_event_type = qe->u.event_type.sub;
+	event->sched_type = sched_type_map[qe->sched_type];
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, 1);
+
+	return 1;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3560,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v8 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c022139..cdabc9b 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -90,17 +90,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3529,6 +3518,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3538,6 +3770,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v8 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index cdabc9b..4e1af0a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1021,6 +1021,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1181,6 +1208,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2681,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2709,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2744,7 +2803,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2752,21 +2811,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2774,7 +2833,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2782,7 +2841,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2790,7 +2849,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2798,7 +2857,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3203,7 +3262,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3373,7 +3433,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3417,6 +3478,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3449,6 +3532,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3468,6 +3552,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3486,6 +3574,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3505,6 +3594,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3811,7 +3904,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3860,6 +3953,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index 892d55f..258d4db 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v8 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1551 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1560 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4e1af0a..0585875 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3878,6 +3878,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..2be5520
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+static inline int
+create_ordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
+}
+
+static inline int
+create_unordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v8 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 0585875..aa22d03 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -158,6 +158,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3854,6 +3857,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3868,7 +3893,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v8 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index aa22d03..b21c9b1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3879,6 +3879,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3900,6 +3912,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v9 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in v9 
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
Depends-on: patch-79539 ("eal: add new x86 cpuid support for WAITPKG")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4129 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1551 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21764 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v9 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v9 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v9 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v9 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v9 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..7bbe69e
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__v2di src_data0 = (__v2di){_src[0], _src[1]};
+
+	__builtin_ia32_movntdq((__v2di *)dest, (__v2di)src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v9 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v9 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v9 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v9 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v9 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v9 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v9 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v9 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanded (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v9 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v9 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v9 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v9 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v9 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 728 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 749 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..c022139 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2812,9 +2812,728 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline bool
+dlb_cq_is_empty(struct dlb_port *qm_port)
+{
+	volatile struct dlb_dequeue_qe *qe_ptr;
+	struct dlb_dequeue_qe qe;
+
+	qe_ptr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	qe = qe_ptr[qm_port->cq_idx];
+
+	return (qe.cq_gen != qm_port->gen_bit);
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+
+		cq_base = port_data->cq_base;
+
+		/* Block on cache line write to CQ. Note: it's
+		 * safe to access the per-process cq_base
+		 * address here, since the PMD has already
+		 * attempted at least one CQ dequeue.
+		 */
+		dlb_umonitor(&cq_base[qm_port->cq_idx]);
+
+		/* Avoid race condition. Check if still empty */
+		if (dlb_cq_is_empty(qm_port)) {
+			dlb_umwait(RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE,
+				   timeout + start_ticks);
+			DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait,
+				     1);
+		}
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_qe(struct dlb_eventdev_port *ev_port __rte_unused,
+		       struct dlb_port *qm_port,
+		       struct rte_event *event,
+		       struct dlb_dequeue_qe *qe)
+{
+	int sched_type_map[4] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+
+	DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+		    (long long)qe->data, qe->qid,
+		    qe->u.event_type.major,
+		    qe->u.event_type.sub,
+		    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+	/* Fill in event information.
+	 * Note that flow_id must be embedded in the data by
+	 * the app, such as the mbuf RSS hash field if the data
+	 * buffer is a mbuf.
+	 */
+	if (unlikely(qe->error)) {
+		DLB_LOG_ERR("QE error bit ON\n");
+		DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+		dlb_consume_qe_immediate(qm_port, 1);
+		return 0;
+	}
+
+	event->u64 = qe->data;
+	event->queue_id = qid_mappings[qe->qid];
+	event->priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+	event->event_type = qe->u.event_type.major;
+	event->sub_event_type = qe->u.event_type.sub;
+	event->sched_type = sched_type_map[qe->sched_type];
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, 1);
+
+	return 1;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3560,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v9 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c022139..cdabc9b 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -90,17 +90,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3529,6 +3518,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3538,6 +3770,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v9 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index cdabc9b..4e1af0a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1021,6 +1021,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1181,6 +1208,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2681,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2709,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2744,7 +2803,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2752,21 +2811,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2774,7 +2833,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2782,7 +2841,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2790,7 +2849,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2798,7 +2857,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3203,7 +3262,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3373,7 +3433,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3417,6 +3478,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3449,6 +3532,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3468,6 +3552,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3486,6 +3574,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3505,6 +3594,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3811,7 +3904,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3860,6 +3953,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v9 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1551 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1560 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4e1af0a..0585875 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3878,6 +3878,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..2be5520
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+static inline int
+create_ordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
+}
+
+static inline int
+create_unordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v9 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 0585875..aa22d03 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -158,6 +158,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3854,6 +3857,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3868,7 +3893,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v9 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index aa22d03..b21c9b1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3879,6 +3879,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3900,6 +3912,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v10 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128() 
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
Depends-on: patch-79539 ("eal: add new x86 cpuid support for WAITPKG")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4092 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21715 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v10 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 01/23] event/dlb: add documentation and meson infrastructure

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:27 PM
> To: Thomas Monjalon <thomas@monjalon.net>; Richardson, Bruce
> <bruce.richardson@intel.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> <nhorman@tuxdriver.com>
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com
> Subject: [PATCH v10 01/23] event/dlb: add documentation and meson
> infrastructure
> 
> Note that config/rte_config.h contains several configuration
> switches, providing for fine control of the PMD's
> runtime behaviour.
> 
> The meson infrastructure is expanded as additional files are
> added to this patchset.
> 
> Adds announcement of availabililty of the new driver
> for Intel Dynamic Load Balancer 1.0 hardware.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

[dpdk-dev] [PATCH v10 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v10 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v10 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v10 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..9f30aa2
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__v2di src_data0 = (__v2di){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v10 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 06/23] event/dlb: add eventdev probe

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:27 PM
> To: Burakov, Anatoly <anatoly.burakov@intel.com>
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 06/23] event/dlb: add eventdev probe
> 
> Add the eventdev portion of probe, and parse command line
> options, but do not initialize hardware.
> 
> Changes since v5 patch-set probe:
> 
> Primary and secondary probe-time init has been removed, and
> will be introduced in subsequent patches contained in
> this patch-set.
> 
> Hardware init has been moved to a subsequent patch in order to
> minimize the patch size.
> 
> Initialization of the flexible interface layer has been moved to
> a subsequent patch in order to minimize patch size.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> ---

Same as with the 2.0 patchset: if there's a next version, please put the
'Changes since' info below the '---' line, so it's not included in the commit
message. (Otherwise this can get fixed during merge.)

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

Re: [dpdk-dev] [PATCH v10 06/23] event/dlb: add eventdev probe

[McDaniel, Timothy]

> -----Original Message-----
> From: Eads, Gage <gage.eads@intel.com>
> Sent: Friday, October 30, 2020 2:51 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>; Burakov, Anatoly
> <anatoly.burakov@intel.com>
> Cc: dev@dpdk.org; Carrillo, Erik G <Erik.G.Carrillo@intel.com>; Van Haaren,
> Harry <harry.van.haaren@intel.com>; jerinj@marvell.com;
> thomas@monjalon.net
> Subject: RE: [PATCH v10 06/23] event/dlb: add eventdev probe
> 
> 
> 
> > -----Original Message-----
> > From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> > Sent: Friday, October 30, 2020 1:27 PM
> > To: Burakov, Anatoly <anatoly.burakov@intel.com>
> > Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> > <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> > jerinj@marvell.com; thomas@monjalon.net
> > Subject: [PATCH v10 06/23] event/dlb: add eventdev probe
> >
> > Add the eventdev portion of probe, and parse command line
> > options, but do not initialize hardware.
> >
> > Changes since v5 patch-set probe:
> >
> > Primary and secondary probe-time init has been removed, and
> > will be introduced in subsequent patches contained in
> > this patch-set.
> >
> > Hardware init has been moved to a subsequent patch in order to
> > minimize the patch size.
> >
> > Initialization of the flexible interface layer has been moved to
> > a subsequent patch in order to minimize patch size.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > ---
> 
> Same as with the 2.0 patchset: if there's a next version, please put the
> 'Changes since' info below the '---' line, so it's not included in the commit
> message. (Otherwise this can get fixed during merge.)
> 
> Reviewed-by: Gage Eads <gage.eads@intel.com>
> 
> Thanks,
> Gage

Thanks Gage.
Had the opportunity, so I fixed it.

[dpdk-dev] [PATCH v10 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 07/23] event/dlb: add flexible interface

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:27 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 07/23] event/dlb: add flexible interface
> 
> This commit introduces the flexible interface. This
> interface allows the core code to operate in PF mode (direct
> hardware access) or bifurcated mode (hardware configured via
> kernel driver). This driver currently only supports PF modei,
> but bifurcated mode will be added in a future patch-set.
> Note that the flexible interface is not used for data path
> operations, and thus there are no performance concerns
> related to the use of function pointers.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

[dpdk-dev] [PATCH v10 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 08/23] event/dlb: add probe-time hardware init

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:27 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 08/23] event/dlb: add probe-time hardware init
> 
> This commit adds probe-time low level hardware
> initialization.  It also adds probe-time init for both
> primary and secondary DPDK processes.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

[dpdk-dev] [PATCH v10 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v10 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v10 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v10 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v10 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanded (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 13/23] event/dlb: add port setup

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:27 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 13/23] event/dlb: add port setup
> 
> Configure the load balanded (ldb) or directed (dir) port.
> The consumer queue (CQ) and producer port (PP) are also
> set up here.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

Here also, typo in the commit message ("balanded" -> "balanced"), but that
can be fixed in the merge.

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

Re: [dpdk-dev] [PATCH v10 13/23] event/dlb: add port setup

[McDaniel, Timothy]

> -----Original Message-----
> From: Eads, Gage <gage.eads@intel.com>
> Sent: Friday, October 30, 2020 2:51 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev@dpdk.org; Carrillo, Erik G <Erik.G.Carrillo@intel.com>; Van Haaren,
> Harry <harry.van.haaren@intel.com>; jerinj@marvell.com;
> thomas@monjalon.net
> Subject: RE: [PATCH v10 13/23] event/dlb: add port setup
> 
> 
> 
> > -----Original Message-----
> > From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> > Sent: Friday, October 30, 2020 1:27 PM
> > Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> > <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> > jerinj@marvell.com; thomas@monjalon.net
> > Subject: [PATCH v10 13/23] event/dlb: add port setup
> >
> > Configure the load balanded (ldb) or directed (dir) port.
> > The consumer queue (CQ) and producer port (PP) are also
> > set up here.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> 
> Here also, typo in the commit message ("balanded" -> "balanced"), but that
> can be fixed in the merge.
> 
> Reviewed-by: Gage Eads <gage.eads@intel.com>
> 
> Thanks,
> Gage

Thanks Gage.
Had the opportunity, so I fixed it.

[dpdk-dev] [PATCH v10 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v10 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v10 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v10 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v10 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 691 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 712 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..a30d270 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2812,9 +2813,690 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline bool
+dlb_cq_is_empty(struct dlb_port *qm_port)
+{
+	volatile struct dlb_dequeue_qe *qe_ptr;
+	struct dlb_dequeue_qe qe;
+
+	qe_ptr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	qe = qe_ptr[qm_port->cq_idx];
+
+	return (qe.cq_gen != qm_port->gen_bit);
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3523,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 18/23] event/dlb: add dequeue and its burst variants

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:28 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 18/23] event/dlb: add dequeue and its burst variants
> 
> Add support for dequeue, dequeue_burst, ...
> 
> DLB does not currently support interrupts, but instead uses
> umonitor/umwait if supported by the processor. This allows
> the software to monitor and wait on writes to a cache-line.
> 
> DLB supports normal and sparse cq mode. In normal mode the
> hardware will pack 4 QEs into each cache line. In sparse cq
> mode, the hardware will only populate one QE per cache line.
> Software must be aware of the cq mode, and take the appropriate
> actions, based on the mode.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

I believe I added my tag to the v5 of this patch, but for good measure:
Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

[dpdk-dev] [PATCH v10 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index a30d270..3ba851a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3492,6 +3481,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3501,6 +3733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v10 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 3ba851a..d6f1575 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1022,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1182,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2682,7 +2711,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2710,6 +2740,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2745,7 +2804,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2753,21 +2812,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2775,7 +2834,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2783,7 +2842,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2791,7 +2850,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2799,7 +2858,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3211,7 +3270,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3336,7 +3396,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3380,6 +3441,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3412,6 +3495,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3431,6 +3515,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3449,6 +3537,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3468,6 +3557,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3774,7 +3867,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3823,6 +3916,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v10 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index d6f1575..24f3ecd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3841,6 +3841,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..aefadab
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v10 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 24f3ecd..c37290e 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3817,6 +3820,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3831,7 +3856,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

Re: [dpdk-dev] [PATCH v10 22/23] event/dlb: add queue and port release

[Eads, Gage]

> -----Original Message-----
> From: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Sent: Friday, October 30, 2020 1:28 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net
> Subject: [PATCH v10 22/23] event/dlb: add queue and port release
> 
> These entry points are NO-OPS. DLB does not support
> reconfiguring individual queues or ports. The entire device
> must be reconfigured.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

Reviewed-by: Gage Eads <gage.eads@intel.com>

Thanks,
Gage

[dpdk-dev] [PATCH v10 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c37290e..68e9bd8 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3842,6 +3842,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3863,6 +3875,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v11 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21703 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v11 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v11 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v11 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v11 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v11 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..9f30aa2
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__v2di src_data0 = (__v2di){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v11 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v11 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v11 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v11 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v11 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v11 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v11 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v11 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v11 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v11 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v11 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v11 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v11 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..e2f769c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2812,9 +2813,678 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3511,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v11 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e2f769c..a8efcc1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v11 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index a8efcc1..31e4d50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1022,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1182,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2682,7 +2711,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2710,6 +2740,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2745,7 +2804,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2753,21 +2812,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2775,7 +2834,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2783,7 +2842,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2791,7 +2850,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2799,7 +2858,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3199,7 +3258,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3324,7 +3384,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3368,6 +3429,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3400,6 +3483,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3419,6 +3503,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3437,6 +3525,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3456,6 +3545,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3762,7 +3855,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3811,6 +3904,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v11 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 31e4d50..c37de85 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..aefadab
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v11 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c37de85..41ccb4a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v11 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 41ccb4a..4aab9f6 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v12 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with
  _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")


Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21703 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v12 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v12 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v12 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v12 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v12 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..82edbb2
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v12 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v12 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v12 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v12 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v12 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v12 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v12 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v12 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v12 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v12 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v12 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v12 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v12 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..e2f769c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2812,9 +2813,678 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3511,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v12 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e2f769c..a8efcc1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v12 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index a8efcc1..31e4d50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1022,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1182,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2682,7 +2711,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2710,6 +2740,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2745,7 +2804,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2753,21 +2812,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2775,7 +2834,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2783,7 +2842,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2791,7 +2850,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2799,7 +2858,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3199,7 +3258,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3324,7 +3384,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3368,6 +3429,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3400,6 +3483,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3419,6 +3503,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3437,6 +3525,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3456,6 +3545,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3762,7 +3855,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3811,6 +3904,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v12 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 31e4d50..c37de85 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..aefadab
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v12 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c37de85..41ccb4a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v12 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 41ccb4a..4aab9f6 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v13 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with
  _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   40 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21684 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v13 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v13 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v13 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v13 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v13 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..1ecddb7
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

[dpdk-dev] [PATCH v13 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v13 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v13 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v13 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v13 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v13 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v13 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v13 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v13 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v13 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v13 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v13 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..4d65a7f 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2135,6 +2154,664 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2836,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v13 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d65a7f..e2f769c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2812,9 +2813,678 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2841,6 +3511,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v13 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e2f769c..a8efcc1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v13 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..1c83bf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index a8efcc1..31e4d50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1022,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1182,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2682,7 +2711,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2710,6 +2740,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2745,7 +2804,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2753,21 +2812,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2775,7 +2834,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2783,7 +2842,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2791,7 +2850,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2799,7 +2858,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3199,7 +3258,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3324,7 +3384,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3368,6 +3429,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3400,6 +3483,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3419,6 +3503,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3437,6 +3525,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3456,6 +3545,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3762,7 +3855,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3811,6 +3904,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v13 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 31e4d50..c37de85 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..aefadab
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v13 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c37de85..41ccb4a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v13 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 41ccb4a..4aab9f6 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD

[Jerin Jacob]

On Sat, Oct 31, 2020 at 7:41 AM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> The following patch series adds support for a new eventdev PMD. The DLB
> PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> The DLB is a PCIe device that provides load-balanced, prioritized
> scheduling of core-to-core communication. The device consists of
> queues and arbiters that connect producer and consumer cores, and
> implements load-balanced queueing features including:
> - Lock-free multi-producer/multi-consumer operation.
> - Multiple priority levels for varying traffic types.
> - 'Direct' traffic (i.e. multi-producer/single-consumer)
> - Simple unordered load-balanced distribution.
> - Atomic lock-free load balancing across multiple consumers.
> - Queue element reordering feature allowing ordered load-balanced
>   distribution.
>
> The DLB hardware supports both load balanced and directed ports and
> queues. Unlike other eventdev devices already in the repo,  not all
> DLB ports and queues are equally capable. In particular, directed
> ports are limited to a single link, and must be connected to a directed
> queue.
> Additionally, even though LDB ports may link multiple queues, the
> number of queues that may be linked is limited by hardware. Another
> difference is that DLB does not have a straightforward way of carrying
> the flow_id in the queue elements (QE) that the hardware operates on.
>
> While reviewing the code, please be aware that this PMD has full
> control over the DLB hardware. Intel will be extending the DLB PMD
> in the future (not as part of this first series) with a mode that we
> refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> kernel driver to configure the device, ports, and queues, and memory
> maps device MMIO so datapath operations occur purely in user-space.
>
> The framework to support both the PF PMD and bifurcated PMD exists in
> this patchset, and is why the iface.[ch] layer is present.
>
> Major changes in V13
> ====================
> - removed now unused functions dlb_umwait and dlb_umonitor

build error with clang at "event/dlb: add enqueue and its burst
variants" patch. Please make sure each patch builds to avoid delay in
merging the patch.
Also, address the David comment on the doc for the next version.


FAILED: drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o
ccache clang -Idrivers/libtmp_rte_event_dlb.a.p -Idrivers -I../drivers
-Idrivers/event/dlb -I../drivers/event/dlb -Ilib/librte_eventdev
-I../lib/librte_eventdev -I. -I.. -Iconfig -I../config
-Ilib/librte_eal/include -I../lib/librte_eal/incl
ude -Ilib/librte_eal/linux/include -I../lib/librte_eal/linux/include
-Ilib/librte_eal/x86/include -I../lib/librte_eal/x86/include
-Ilib/librte_eal/common -I../lib/librte_eal/common -Ilib/librte_eal
-I../lib/librte_eal -Ilib/librte_kvargs -I
../lib/librte_kvargs -Ilib/librte_metrics -I../lib/librte_metrics
-Ilib/librte_telemetry -I../lib/librte_telemetry -Ilib/librte_ring
-I../lib/librte_ring -Ilib/librte_ethdev -I../lib/librte_ethdev
-Ilib/librte_net -I../lib/librte_net -Ilib/
librte_mbuf -I../lib/librte_mbuf -Ilib/librte_mempool
-I../lib/librte_mempool -Ilib/librte_meter -I../lib/librte_meter
-Ilib/librte_hash -I../lib/librte_hash -Ilib/librte_rcu
-I../lib/librte_rcu -Ilib/librte_timer -I../lib/librte_timer -Ili
b/librte_cryptodev -I../lib/librte_cryptodev -Ilib/librte_pci
-I../lib/librte_pci -Idrivers/bus/pci -I../drivers/bus/pci
-I../drivers/bus/pci/linux -Xclang -fcolor-diagnostics -pipe
-D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g
-include rte_config.h -Wextra -Wcast-qual -Wdeprecated
-Wformat-nonliteral -Wformat-security -Wmissing-declarations
-Wmissing-prototypes -Wnested-externs -Wold-style-definition
-Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwr
ite-strings -Wno-address-of-packed-member
-Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native
-DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -MD -MQ
drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -MF
drivers/libtmp_rte_even
t_dlb.a.p/event_dlb_dlb.c.o.d -o
drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -c
../drivers/event/dlb/dlb.c
../drivers/event/dlb/dlb.c:2777:1: error: unused function
'dlb_event_enqueue_delayed' [-Werror,-Wunused-function]
dlb_event_enqueue_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2762:1: error: unused function
'dlb_event_enqueue_burst_delayed' [-Werror,-Wunused-function]
dlb_event_enqueue_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2792:1: error: unused function
'dlb_event_enqueue_new_burst_delayed' [-Werror,-Wunused-function]
dlb_event_enqueue_new_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2808:1: error: unused function
'dlb_event_enqueue_forward_burst_delayed' [-Werror,-Wunused-function]
dlb_event_enqueue_forward_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2605:1: error: unused function
'dlb_construct_token_pop_qe' [-Werror,-Wunused-function]
dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
^
../drivers/event/dlb/dlb.c:2653:1: error: unused function
'dlb_consume_qe_immediate' [-Werror,-Wunused-function]
dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
^
6 errors generated.


>
> Major changes in V12
> ====================
> - Fix CENTOS build error: use __m128i instead of __v2di with
>   _mm_stream_si128
>
> Major changes in V11
> ====================
> - removed unused function, fixing build error
> - fixed typo in port_setup commit message
> - this patch series is based on dpdk-next-eventdev
>
> Major changes in v10
> =====================
> - convert to use rte_power_monitor patches
> - replace __builtin_ia32_movntdq() with _mm_stream_si128()
> - remove unused functions in dlb_selftest.c
>
> Major changes in v9
> =====================
> - fixed a build error due to __rte_cache_aligned being placed after
>   the ";" character, instead of before it.
>
> Major changes in v8 after dpdk reviews
> =====================
> - moved introduction of dlb in relnotes_20_11 to first patch in series
> - fixed underlines in dlb.rst that were too short
> - note that the code still uses its private byte-encoded versions of
>   umonitor/umwait, rather than the new functions in the power
>   patch that are built on top of those intrinsics. This is intentional.
>
> Major changes in v7 after dpdk reviews
> =====================
> - updated MAINTAINERS file to alphabetically insert DLB
> - don't create RTE_ symbols in PMD
> - converted to use version.map scheme
> - converted to use .._master_lcore instead of .._main_lcore
> - this patch set is based on dpdk-next-eventdev
>
> Major changes in v6 after dpdk reviews:
> =====================
> - fixed meson conditional build. Moved test into driver’s meson.build
>   file instead of event/meson.build
> - documentation is populated as associated code is introduced
> - add log_register in add dynamic logging patch
> - rename RTE_xxx symbol(s) as DLB2_xxx
> - replaced function ptr enqueue_four with direct call to movdir64b
> - remove unused port_pages
> - broke up probe patch into 3 smaller patches for easier review
> - changed param order of movdir64b/movntdq to match intrinsics
> - added self to MAINTAINERS files
> - squashed announcement of availability into last patch in series
> - correct spelling errors and delete repeated words
> - DPDK_21.0 -> DPDK 21 in map file
> - add experimental banner to public structs and APIs
> - implemented other suggestions from code reviews of DLB2 PMD. The
>   software is very similar in form so some DLB2 reviews comments
>   were applicable to DLB as well
>
> Major changes in v5 after dpdk reviews and additional internal reviews
> by colleagues at Intel:
> ================
> - implement changes requested in code reviews by Gage Eads and Mike Chen
> - fix a memzone leak
> - convert to use eal rte-cpuflags patch from Liang Ma
>
> Major changes in v4 after dpdk reviews and additional internal reviews
> by colleagues at Intel:
> ================
> - Remove make infrastructure
> - shared code (pf/base) is now added incrementally
> - flexible interface (iface.[ch]) is now added incrementally
> - removed calls to rte_panic
> - do not call pthread_create directly
> - remove unused internal API, os_time
> - convert rte_atomic to __atomic builtins
> - broke out eventdev ABI changes, test/api changes, and new internal PCI
>   named probe API
> - relocated enqueue logic to enqueue patch
>
> Major Changes in V3:
> ================
> - Fixed a memory corruption issue due to not allocating enough CQ
> memory for depths < 8. Hardware requires minimum allocation to be
> at least 8 entries.
> - Address review comments from Gage and Mattias.
> - Remove versioning
> - minor formatting changes
>
> Major changes in V2:
> ================
> - Correct ABI break that was present in V1.
> - Address some of the review comments received from Mattias.
>   I will address the remaining items identified by Mattias in the next
>   patch delivery.
> - General code cleanup based on internal code reviews
>
> Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
>
> Timothy McDaniel (23):
>   event/dlb: add documentation and meson infrastructure
>   event/dlb: add dynamic logging
>   event/dlb: add private data structures and constants
>   event/dlb: add definitions shared with LKM or shared code
>   event/dlb: add inline functions
>   event/dlb: add eventdev probe
>   event/dlb: add flexible interface
>   event/dlb: add probe-time hardware init
>   event/dlb: add xstats
>   event/dlb: add infos get and configure
>   event/dlb: add queue and port default conf
>   event/dlb: add queue setup
>   event/dlb: add port setup
>   event/dlb: add port link
>   event/dlb: add port unlink and port unlinks in progress
>   event/dlb: add eventdev start
>   event/dlb: add enqueue and its burst variants
>   event/dlb: add dequeue and its burst variants
>   event/dlb: add eventdev stop and close
>   event/dlb: add PMD's token pop public interface
>   event/dlb: add PMD self-tests
>   event/dlb: add queue and port release
>   event/dlb: add timeout ticks entry point
>
>  MAINTAINERS                                  |    6 +-
>  app/test/test_eventdev.c                     |    7 +
>  config/rte_config.h                          |    6 +
>  doc/api/doxy-api-index.md                    |    1 +
>  doc/guides/eventdevs/dlb.rst                 |  341 ++
>  doc/guides/eventdevs/index.rst               |    1 +
>  doc/guides/rel_notes/release_20_11.rst       |    5 +
>  drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
>  drivers/event/dlb/dlb_iface.c                |   79 +
>  drivers/event/dlb/dlb_iface.h                |   82 +
>  drivers/event/dlb/dlb_inline_fns.h           |   40 +
>  drivers/event/dlb/dlb_log.h                  |   25 +
>  drivers/event/dlb/dlb_priv.h                 |  513 ++
>  drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
>  drivers/event/dlb/dlb_user.h                 |  814 +++
>  drivers/event/dlb/dlb_xstats.c               | 1222 +++++
>  drivers/event/dlb/meson.build                |   21 +
>  drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
>  drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
>  drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
>  drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
>  drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
>  drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
>  drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
>  drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
>  drivers/event/dlb/pf/dlb_main.c              |  586 +++
>  drivers/event/dlb/pf/dlb_main.h              |   47 +
>  drivers/event/dlb/pf/dlb_pf.c                |  750 +++
>  drivers/event/dlb/rte_pmd_dlb.c              |   38 +
>  drivers/event/dlb/rte_pmd_dlb.h              |   77 +
>  drivers/event/dlb/version.map                |    9 +
>  drivers/event/meson.build                    |    2 +-
>  32 files changed, 21684 insertions(+), 2 deletions(-)
>  create mode 100644 doc/guides/eventdevs/dlb.rst
>  create mode 100644 drivers/event/dlb/dlb.c
>  create mode 100644 drivers/event/dlb/dlb_iface.c
>  create mode 100644 drivers/event/dlb/dlb_iface.h
>  create mode 100644 drivers/event/dlb/dlb_inline_fns.h
>  create mode 100644 drivers/event/dlb/dlb_log.h
>  create mode 100644 drivers/event/dlb/dlb_priv.h
>  create mode 100644 drivers/event/dlb/dlb_selftest.c
>  create mode 100644 drivers/event/dlb/dlb_user.h
>  create mode 100644 drivers/event/dlb/dlb_xstats.c
>  create mode 100644 drivers/event/dlb/meson.build
>  create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
>  create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
>  create mode 100644 drivers/event/dlb/pf/dlb_main.c
>  create mode 100644 drivers/event/dlb/pf/dlb_main.h
>  create mode 100644 drivers/event/dlb/pf/dlb_pf.c
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
>  create mode 100644 drivers/event/dlb/version.map
>
> --
> 2.6.4
>

Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD

[McDaniel, Timothy]

> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> Sent: Saturday, October 31, 2020 7:50 AM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dpdk-dev <dev@dpdk.org>; Carrillo, Erik G <Erik.G.Carrillo@intel.com>;
> Eads, Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob <jerinj@marvell.com>; Thomas
> Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD
> 
> On Sat, Oct 31, 2020 at 7:41 AM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > The following patch series adds support for a new eventdev PMD. The DLB
> > PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> > The DLB is a PCIe device that provides load-balanced, prioritized
> > scheduling of core-to-core communication. The device consists of
> > queues and arbiters that connect producer and consumer cores, and
> > implements load-balanced queueing features including:
> > - Lock-free multi-producer/multi-consumer operation.
> > - Multiple priority levels for varying traffic types.
> > - 'Direct' traffic (i.e. multi-producer/single-consumer)
> > - Simple unordered load-balanced distribution.
> > - Atomic lock-free load balancing across multiple consumers.
> > - Queue element reordering feature allowing ordered load-balanced
> >   distribution.
> >
> > The DLB hardware supports both load balanced and directed ports and
> > queues. Unlike other eventdev devices already in the repo,  not all
> > DLB ports and queues are equally capable. In particular, directed
> > ports are limited to a single link, and must be connected to a directed
> > queue.
> > Additionally, even though LDB ports may link multiple queues, the
> > number of queues that may be linked is limited by hardware. Another
> > difference is that DLB does not have a straightforward way of carrying
> > the flow_id in the queue elements (QE) that the hardware operates on.
> >
> > While reviewing the code, please be aware that this PMD has full
> > control over the DLB hardware. Intel will be extending the DLB PMD
> > in the future (not as part of this first series) with a mode that we
> > refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> > kernel driver to configure the device, ports, and queues, and memory
> > maps device MMIO so datapath operations occur purely in user-space.
> >
> > The framework to support both the PF PMD and bifurcated PMD exists in
> > this patchset, and is why the iface.[ch] layer is present.
> >
> > Major changes in V13
> > ====================
> > - removed now unused functions dlb_umwait and dlb_umonitor
> 
> build error with clang at "event/dlb: add enqueue and its burst
> variants" patch. Please make sure each patch builds to avoid delay in
> merging the patch.
> Also, address the David comment on the doc for the next version.
> 
> 
> FAILED: drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o
> ccache clang -Idrivers/libtmp_rte_event_dlb.a.p -Idrivers -I../drivers
> -Idrivers/event/dlb -I../drivers/event/dlb -Ilib/librte_eventdev
> -I../lib/librte_eventdev -I. -I.. -Iconfig -I../config
> -Ilib/librte_eal/include -I../lib/librte_eal/incl
> ude -Ilib/librte_eal/linux/include -I../lib/librte_eal/linux/include
> -Ilib/librte_eal/x86/include -I../lib/librte_eal/x86/include
> -Ilib/librte_eal/common -I../lib/librte_eal/common -Ilib/librte_eal
> -I../lib/librte_eal -Ilib/librte_kvargs -I
> ../lib/librte_kvargs -Ilib/librte_metrics -I../lib/librte_metrics
> -Ilib/librte_telemetry -I../lib/librte_telemetry -Ilib/librte_ring
> -I../lib/librte_ring -Ilib/librte_ethdev -I../lib/librte_ethdev
> -Ilib/librte_net -I../lib/librte_net -Ilib/
> librte_mbuf -I../lib/librte_mbuf -Ilib/librte_mempool
> -I../lib/librte_mempool -Ilib/librte_meter -I../lib/librte_meter
> -Ilib/librte_hash -I../lib/librte_hash -Ilib/librte_rcu
> -I../lib/librte_rcu -Ilib/librte_timer -I../lib/librte_timer -Ili
> b/librte_cryptodev -I../lib/librte_cryptodev -Ilib/librte_pci
> -I../lib/librte_pci -Idrivers/bus/pci -I../drivers/bus/pci
> -I../drivers/bus/pci/linux -Xclang -fcolor-diagnostics -pipe
> -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g
> -include rte_config.h -Wextra -Wcast-qual -Wdeprecated
> -Wformat-nonliteral -Wformat-security -Wmissing-declarations
> -Wmissing-prototypes -Wnested-externs -Wold-style-definition
> -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwr
> ite-strings -Wno-address-of-packed-member
> -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native
> -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -MD -MQ
> drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -MF
> drivers/libtmp_rte_even
> t_dlb.a.p/event_dlb_dlb.c.o.d -o
> drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -c
> ../drivers/event/dlb/dlb.c
> ../drivers/event/dlb/dlb.c:2777:1: error: unused function
> 'dlb_event_enqueue_delayed' [-Werror,-Wunused-function]
> dlb_event_enqueue_delayed(void *event_port,
> ^
> ../drivers/event/dlb/dlb.c:2762:1: error: unused function
> 'dlb_event_enqueue_burst_delayed' [-Werror,-Wunused-function]
> dlb_event_enqueue_burst_delayed(void *event_port,
> ^
> ../drivers/event/dlb/dlb.c:2792:1: error: unused function
> 'dlb_event_enqueue_new_burst_delayed' [-Werror,-Wunused-function]
> dlb_event_enqueue_new_burst_delayed(void *event_port,
> ^
> ../drivers/event/dlb/dlb.c:2808:1: error: unused function
> 'dlb_event_enqueue_forward_burst_delayed' [-Werror,-Wunused-function]
> dlb_event_enqueue_forward_burst_delayed(void *event_port,
> ^
> ../drivers/event/dlb/dlb.c:2605:1: error: unused function
> 'dlb_construct_token_pop_qe' [-Werror,-Wunused-function]
> dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
> ^
> ../drivers/event/dlb/dlb.c:2653:1: error: unused function
> 'dlb_consume_qe_immediate' [-Werror,-Wunused-function]
> dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
> ^
> 6 errors generated.
> 

Took care of these in V14.  Hoping it’s the last of the errors.

[dpdk-dev] [PATCH v14 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V14
====================
- Fixed format errors in doc/api/doxy-api-index.md
- Delayed introduction of dlb2_consume_qe_immediate until
  add-dequeue-and-its-burst-variants.patch
- Delayed introduction of dlb2_construct_token_pop_qe until 
  add-PMD-s-token-pop-public-interface.patch
- Delayed introduction of dlb_equeue_*_delayed until
  add dequeue and its burst variants.patch

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with
  _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    3 +-
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4079 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   40 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21684 insertions(+), 3 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v14 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availabililty of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index a3d1927..b904132 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index d8ac359..0a95bf0 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..5324043
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 01/23] event/dlb: add documentation and meson infrastructure

[David Marchand]

On Sat, Oct 31, 2020 at 7:16 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> Note that config/rte_config.h contains several configuration
> switches, providing for fine control of the PMD's
> runtime behaviour.
>
> The meson infrastructure is expanded as additional files are
> added to this patchset.
>
> Adds announcement of availabililty of the new driver

availability*

> for Intel Dynamic Load Balancer 1.0 hardware.
>
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> ---
>  MAINTAINERS                            |  6 +++++-
>  config/rte_config.h                    |  6 ++++++
>  doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
>  doc/guides/eventdevs/index.rst         |  1 +
>  doc/guides/rel_notes/release_20_11.rst |  5 +++++
>  drivers/event/dlb/meson.build          | 13 ++++++++++++
>  drivers/event/dlb/version.map          |  3 +++
>  drivers/event/meson.build              |  2 +-
>  8 files changed, 70 insertions(+), 2 deletions(-)
>  create mode 100644 doc/guides/eventdevs/dlb.rst
>  create mode 100644 drivers/event/dlb/meson.build
>  create mode 100644 drivers/event/dlb/version.map
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index a3d1927..b904132 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
>  M: Pavan Nikhilesh <pbhagavatula@marvell.com>
>  F: drivers/event/octeontx/timvf_*
>
> +Intel DLB
> +M: Timothy McDaniel <timothy.mcdaniel@intel.com>
> +F: drivers/event/dlb/
> +F: doc/guides/eventdevs/dlb.rst
> +
>  Marvell OCTEON TX2
>  M: Pavan Nikhilesh <pbhagavatula@marvell.com>
>  M: Jerin Jacob <jerinj@marvell.com>
> @@ -1198,7 +1203,6 @@ M: Peter Mccarthy <peter.mccarthy@intel.com>
>  F: drivers/event/opdl/
>  F: doc/guides/eventdevs/opdl.rst
>
> -

Please leave this empty line.


>  Rawdev Drivers
>  --------------
>
> diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
> index d8ac359..0a95bf0 100644
> --- a/doc/guides/rel_notes/release_20_11.rst
> +++ b/doc/guides/rel_notes/release_20_11.rst
> @@ -55,6 +55,11 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>
> +* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
> +
> +  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
> +  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
> +
>  * **Added write combining store APIs.**
>

eventdev drivers go after net drivers.


>    Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
> diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
> new file mode 100644
> index 0000000..5324043
> --- /dev/null
> +++ b/drivers/event/dlb/meson.build
> @@ -0,0 +1,13 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2019-2020 Intel Corporation
> +
> +if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
> +        build = false
> +        reason = 'only supported on ARCH_X86_64 Linux'
> +        subdir_done()
> +endif
> +
> +sources = files(
> +)

If you go with empty list, you can also start with headers = files()
(rather than install_headers, see later comment).

> +
> +deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']

Odd to see dependencies with nothing to build.



-- 
David Marchand

[dpdk-dev] [PATCH v14 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5324043..1e7d5ad 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

[dpdk-dev] [PATCH v14 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v14 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..1ecddb7
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[David Marchand]

On Sat, Oct 31, 2020 at 7:17 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> Add miscellaneous inline functions that may be called
> from multiple files.  These functions include inline
> assembly of new x86 instructions, such as movdir64b,
> since they are not available as builtin functions in
> the minimum supported GCC version.
>
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> Reviewed-by: Gage Eads <gage.eads@intel.com>
> ---
>  drivers/event/dlb/dlb_inline_fns.h | 40 ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 40 insertions(+)
>  create mode 100644 drivers/event/dlb/dlb_inline_fns.h
>
> diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
> new file mode 100644
> index 0000000..1ecddb7
> --- /dev/null
> +++ b/drivers/event/dlb/dlb_inline_fns.h
> @@ -0,0 +1,40 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2016-2020 Intel Corporation
> + */
> +

Missing banners, like in dlb2 header (caught by diffing with it).

#ifndef __DLB_INLINE_FNS_H__
etc...

> +#include "rte_memcpy.h"
> +#include "rte_io.h"
> +
> +/* Inline functions required in more than one source file. */
> +
> +static inline struct dlb_eventdev *
> +dlb_pmd_priv(const struct rte_eventdev *eventdev)
> +{
> +       return eventdev->data->dev_private;
> +}
> +
> +static inline void
> +dlb_movntdq_single(void *dest, void *src)
> +{
> +       long long *_src  = (long long *)src;
> +       __m128i src_data0 = (__m128i){_src[0], _src[1]};
> +
> +       _mm_stream_si128(dest, src_data0);
> +}
> +
> +static inline void
> +dlb_cldemote(void *addr)
> +{
> +       /* Load addr into RSI, then demote the cache line of the address
> +        * contained in that register.
> +        */
> +       asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
> +}

Use EAL API.


> +
> +static inline void
> +dlb_movdir64b(void *dest, void *src)
> +{
> +       asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
> +                    :
> +                    : "a" (dest), "d" (src));
> +}

NO!
We introduced stuff in EAL for this, please double check.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 4:54 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions
> 
> On Sat, Oct 31, 2020 at 7:17 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > Add miscellaneous inline functions that may be called
> > from multiple files.  These functions include inline
> > assembly of new x86 instructions, such as movdir64b,
> > since they are not available as builtin functions in
> > the minimum supported GCC version.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > Reviewed-by: Gage Eads <gage.eads@intel.com>
> > ---
> >  drivers/event/dlb/dlb_inline_fns.h | 40
> ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 40 insertions(+)
> >  create mode 100644 drivers/event/dlb/dlb_inline_fns.h
> >
> > diff --git a/drivers/event/dlb/dlb_inline_fns.h
> b/drivers/event/dlb/dlb_inline_fns.h
> > new file mode 100644
> > index 0000000..1ecddb7
> > --- /dev/null
> > +++ b/drivers/event/dlb/dlb_inline_fns.h
> > @@ -0,0 +1,40 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2016-2020 Intel Corporation
> > + */
> > +
> 
> Missing banners, like in dlb2 header (caught by diffing with it).
> 
> #ifndef __DLB_INLINE_FNS_H__
> etc...
> 
> > +#include "rte_memcpy.h"
> > +#include "rte_io.h"
> > +
> > +/* Inline functions required in more than one source file. */
> > +
> > +static inline struct dlb_eventdev *
> > +dlb_pmd_priv(const struct rte_eventdev *eventdev)
> > +{
> > +       return eventdev->data->dev_private;
> > +}
> > +
> > +static inline void
> > +dlb_movntdq_single(void *dest, void *src)
> > +{
> > +       long long *_src  = (long long *)src;
> > +       __m128i src_data0 = (__m128i){_src[0], _src[1]};
> > +
> > +       _mm_stream_si128(dest, src_data0);
> > +}
> > +
> > +static inline void
> > +dlb_cldemote(void *addr)
> > +{
> > +       /* Load addr into RSI, then demote the cache line of the address
> > +        * contained in that register.
> > +        */
> > +       asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
> > +}
> 
> Use EAL API.
> 
> 
> > +
> > +static inline void
> > +dlb_movdir64b(void *dest, void *src)
> > +{
> > +       asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
> > +                    :
> > +                    : "a" (dest), "d" (src));
> > +}
> 
> NO!
> We introduced stuff in EAL for this, please double check.
> 
> 
> --
> David Marchand

I do not see a direct single instruction replacement for the following:
dlb_movntdq_single(void *dest, void *src)
{
       long long *_src  = (long long *)src;
       __m128i src_data0 = (__m128i){_src[0], _src[1]};

       _mm_stream_si128(dest, src_data0);
}

The original code used a builting for movntdq, but we switched to _mm_stream_si128
because one of the builds did not have the movntdq builtin.

Tim

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 4:54 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions
> 
> On Sat, Oct 31, 2020 at 7:17 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > Add miscellaneous inline functions that may be called
> > from multiple files.  These functions include inline
> > assembly of new x86 instructions, such as movdir64b,
> > since they are not available as builtin functions in
> > the minimum supported GCC version.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > Reviewed-by: Gage Eads <gage.eads@intel.com>
> > ---
> >  drivers/event/dlb/dlb_inline_fns.h | 40
> ++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 40 insertions(+)
> >  create mode 100644 drivers/event/dlb/dlb_inline_fns.h
> >
> > diff --git a/drivers/event/dlb/dlb_inline_fns.h
> b/drivers/event/dlb/dlb_inline_fns.h
> > new file mode 100644
> > index 0000000..1ecddb7
> > --- /dev/null
> > +++ b/drivers/event/dlb/dlb_inline_fns.h
> > @@ -0,0 +1,40 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2016-2020 Intel Corporation
> > + */
> > +
> 
> Missing banners, like in dlb2 header (caught by diffing with it).
> 
> #ifndef __DLB_INLINE_FNS_H__
> etc...
> 
> > +#include "rte_memcpy.h"
> > +#include "rte_io.h"
> > +
> > +/* Inline functions required in more than one source file. */
> > +
> > +static inline struct dlb_eventdev *
> > +dlb_pmd_priv(const struct rte_eventdev *eventdev)
> > +{
> > +       return eventdev->data->dev_private;
> > +}
> > +
> > +static inline void
> > +dlb_movntdq_single(void *dest, void *src)
> > +{
> > +       long long *_src  = (long long *)src;
> > +       __m128i src_data0 = (__m128i){_src[0], _src[1]};
> > +
> > +       _mm_stream_si128(dest, src_data0);
> > +}
> > +
> > +static inline void
> > +dlb_cldemote(void *addr)
> > +{
> > +       /* Load addr into RSI, then demote the cache line of the address
> > +        * contained in that register.
> > +        */
> > +       asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
> > +}
> 
> Use EAL API.
> 
> 
> > +
> > +static inline void
> > +dlb_movdir64b(void *dest, void *src)
> > +{
> > +       asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
> > +                    :
> > +                    : "a" (dest), "d" (src));
> > +}
> 
> NO!
> We introduced stuff in EAL for this, please double check.
> 
> 
> --
> David Marchand

I also do not see a replacement for the new MOVDIR64B instruction in dpdk-next-eventdev or dpdk main.

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[David Marchand]

> > > +
> > > +static inline void
> > > +dlb_movdir64b(void *dest, void *src)
> > > +{
> > > +       asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
> > > +                    :
> > > +                    : "a" (dest), "d" (src));
> > > +}
> >
> > NO!
> > We introduced stuff in EAL for this, please double check.
> I also do not see a replacement for the new MOVDIR64B instruction in dpdk-next-eventdev or dpdk main.

Ok, what got introduced in EAL is for MOVDIRI.


So here we go with a MOVDIR64B...
Google tells me:
Availability of the MOVDIR64B instruction is indicated by the presence
of the CPUID feature flag MOVDIR64B (bit 28 of the ECX register in
leaf 07H, see “CPUID—CPU Identification” in the Intel® 64 and IA-32
Architectures Software Developer’s Manual, Volume 2A).

I understand that calling this code must be under a check for
RTE_CPUFLAG_MOVDIR64B.
Which I can't find in this patchset.
What did I miss this time?


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Sunday, November 1, 2020 12:01 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions
> 
> > > > +
> > > > +static inline void
> > > > +dlb_movdir64b(void *dest, void *src)
> > > > +{
> > > > +       asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
> > > > +                    :
> > > > +                    : "a" (dest), "d" (src));
> > > > +}
> > >
> > > NO!
> > > We introduced stuff in EAL for this, please double check.
> > I also do not see a replacement for the new MOVDIR64B instruction in dpdk-
> next-eventdev or dpdk main.
> 
> Ok, what got introduced in EAL is for MOVDIRI.
> 
> 
> So here we go with a MOVDIR64B...
> Google tells me:
> Availability of the MOVDIR64B instruction is indicated by the presence
> of the CPUID feature flag MOVDIR64B (bit 28 of the ECX register in
> leaf 07H, see “CPUID—CPU Identification” in the Intel® 64 and IA-32
> Architectures Software Developer’s Manual, Volume 2A).
> 
> I understand that calling this code must be under a check for
> RTE_CPUFLAG_MOVDIR64B.
> Which I can't find in this patchset.
> What did I miss this time?
> 
> 
> --
> David Marchand

Fair enough question. We currently do not check for availability of MOVDIR64B since
every Intel part that includes DLB or DLB2 is guaranteed to have the MOVDIR64B
instruction.

Re: [dpdk-dev] [PATCH v14 05/23] event/dlb: add inline functions

[David Marchand]

On Sun, Nov 1, 2020 at 7:07 PM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
> > Availability of the MOVDIR64B instruction is indicated by the presence
> > of the CPUID feature flag MOVDIR64B (bit 28 of the ECX register in
> > leaf 07H, see “CPUID—CPU Identification” in the Intel® 64 and IA-32
> > Architectures Software Developer’s Manual, Volume 2A).
> >
> > I understand that calling this code must be under a check for
> > RTE_CPUFLAG_MOVDIR64B.
> > Which I can't find in this patchset.
>
> Fair enough question. We currently do not check for availability of MOVDIR64B since
> every Intel part that includes DLB or DLB2 is guaranteed to have the MOVDIR64B
> instruction.

A check at probe time would not hurt, but if you are sure of it, ok.


-- 
David Marchand

[dpdk-dev] [PATCH v14 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 1e7d5ad..b4bdc8b 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v14 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index b4bdc8b..8707d3d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v14 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 8707d3d..9777178 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v14 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..597c3d7
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1222 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (dev == NULL) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 9777178..552ff9d 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 09/23] event/dlb: add xstats

[David Marchand]

On Sat, Oct 31, 2020 at 7:21 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
> +void
> +dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
> +{
> +       struct dlb_eventdev *dlb;
> +       struct dlb_hw_dev *handle;
> +       int i;
> +
> +       if (f == NULL) {
> +               printf("Invalid file pointer\n");

No printf in a driver!

This check should probably be at eventdev layer level.

> +               return;
> +       }
> +
> +       if (dev == NULL) {

dev can't be NULL.


> +               fprintf(f, "Invalid event device\n");
> +               return;
> +       }
> +
> +       dlb = dlb_pmd_priv(dev);
> +
> +       if (dlb == NULL) {

I wonder if this can happen, dev_private is allocated at the same time
the eventdev dev is.

> +               fprintf(f, "DLB Event device cannot be dumped!\n");
> +               return;
> +       }
> +


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 09/23] event/dlb: add xstats

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 5:00 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 09/23] event/dlb: add xstats
> 
> On Sat, Oct 31, 2020 at 7:21 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> > +void
> > +dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
> > +{
> > +       struct dlb_eventdev *dlb;
> > +       struct dlb_hw_dev *handle;
> > +       int i;
> > +
> > +       if (f == NULL) {
> > +               printf("Invalid file pointer\n");
> 
> No printf in a driver!
> 
> This check should probably be at eventdev layer level.
> 
> > +               return;
> > +       }
> > +
> > +       if (dev == NULL) {
> 
> dev can't be NULL.
> 
> 
> > +               fprintf(f, "Invalid event device\n");
> > +               return;
> > +       }
> > +
> > +       dlb = dlb_pmd_priv(dev);
> > +
> > +       if (dlb == NULL) {
> 
> I wonder if this can happen, dev_private is allocated at the same time
> the eventdev dev is.
> 
> > +               fprintf(f, "DLB Event device cannot be dumped!\n");
> > +               return;
> > +       }
> > +
> 
> 
> --
> David Marchand

Addressed in next patchset

[dpdk-dev] [PATCH v14 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v14 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v14 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..edcc6d1 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 12/23] event/dlb: add queue setup

[David Marchand]

On Sat, Oct 31, 2020 at 7:20 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
> +       if (i == DLB_NUM_SN_GROUPS) {
> +               printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
> +                      __func__, sequence_numbers);

No printf in a driver.


> +               return;
> +       }


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 12/23] event/dlb: add queue setup

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 5:02 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 12/23] event/dlb: add queue setup
> 
> On Sat, Oct 31, 2020 at 7:20 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> > +       if (i == DLB_NUM_SN_GROUPS) {
> > +               printf("[%s()] No groups with %d sequence_numbers are available or
> have free slots\n",
> > +                      __func__, sequence_numbers);
> 
> No printf in a driver.
> 
> 
> > +               return;
> > +       }
> 
> 
> --
> David Marchand

Addressed in next patchset

[dpdk-dev] [PATCH v14 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..f106a07 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index edcc6d1..4d91ddd 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 13/23] event/dlb: add port setup

[David Marchand]

On Sat, Oct 31, 2020 at 7:20 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> Configure the load balanced (ldb) or directed (dir) port.
> The consumer queue (CQ) and producer port (PP) are also
> set up here.
>
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> ---
>  doc/guides/eventdevs/dlb.rst             |   40 +
>  drivers/event/dlb/dlb.c                  |  516 ++++++++++-
>  drivers/event/dlb/dlb_iface.c            |   11 +
>  drivers/event/dlb/dlb_iface.h            |   14 +
>  drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
>  drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
>  6 files changed, 2223 insertions(+), 4 deletions(-)
>
> diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
> index d8e936a..f106a07 100644
> --- a/doc/guides/eventdevs/dlb.rst
> +++ b/doc/guides/eventdevs/dlb.rst
> @@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
>  the DLB does not limit the number of flows a queue can track. In the DLB, all
>  load-balanced queues can use the full 16-bit flow ID range.
>
> +Load-balanced and Directed Ports
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Missing one ~.


-- 
David Marchand

[dpdk-dev] [PATCH v14 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4d91ddd..2ad195d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v14 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2ad195d..c64f559 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v14 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c64f559..780ff7d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v14 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++++-
 drivers/event/dlb/dlb.c      | 567 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 729 insertions(+), 1 deletion(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index f106a07..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 780ff7d..f60a2e2 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2135,6 +2135,568 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2721,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v14 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 769 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 790 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index f60a2e2..c227893 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -71,6 +72,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 uint32_t
 dlb_get_queue_depth(struct dlb_eventdev *dlb,
 		    struct dlb_eventdev_queue *queue)
@@ -2605,6 +2625,46 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,
 	dlb_pp_write(qm_port->qe4, port_data);
 }
 
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
@@ -2675,12 +2735,27 @@ dlb_event_enqueue_burst(void *event_port,
 }
 
 static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
 	return __dlb_event_enqueue_burst(event_port, events, 1);
 }
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
 static uint16_t
 dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
@@ -2690,6 +2765,14 @@ dlb_event_enqueue_new_burst(void *event_port,
 }
 
 static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
 dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
@@ -2697,9 +2780,686 @@ dlb_event_enqueue_forward_burst(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2726,6 +3486,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v14 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c227893..8bb3db7 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3455,6 +3444,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3464,6 +3696,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v14 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   3 +-
 drivers/event/dlb/dlb.c         | 145 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 262 insertions(+), 14 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index a9c12d1..2ff3f74 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -51,7 +51,8 @@ The public API headers are grouped by topics:
   [dpaa2_mempool]      (@ref rte_dpaa2_mempool.h),
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
-  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h),
+  [dlb]                (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8bb3db7..12cf38b 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1022,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1182,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2591,6 +2620,30 @@ dlb_event_build_hcws(struct dlb_port *qm_port,
 	}
 }
 
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+	qm_port->owed_tokens = 0;
+}
+
 static __rte_always_inline void
 dlb_pp_write(struct dlb_enqueue_qe *qe4,
 	     struct process_local_port_data *port_data)
@@ -2657,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2685,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2720,7 +2803,7 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static inline uint16_t
@@ -2728,21 +2811,21 @@ dlb_event_enqueue_burst_delayed(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
 }
 
 static inline uint16_t
 dlb_event_enqueue_delayed(void *event_port,
 			  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2750,7 +2833,7 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2758,7 +2841,7 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 				    const struct rte_event events[],
 				    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2766,7 +2849,7 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
 }
 
 static uint16_t
@@ -2774,7 +2857,7 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3174,7 +3257,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3299,7 +3383,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3343,6 +3428,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3375,6 +3482,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3394,6 +3502,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3412,6 +3524,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3431,6 +3544,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3737,7 +3854,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3786,6 +3903,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 552ff9d..7f38c30 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 20/23] event/dlb: add PMD's token pop public interface

[David Marchand]

On Sat, Oct 31, 2020 at 7:23 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> The PMD uses a public interface to allow applications to
> control the token pop mode. Supported token pop modes are
> as follows, and they impact core scheduling affinity for
> ldb ports.
>
> AUTO_POP: Pop the CQ tokens immediately after dequeueing.
> DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
>              are released. Supported on load-balanced ports
>              only.
> DEFERRED_POP: Pop the CQ tokens during next dequeue operation.
>
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> Reviewed-by: Gage Eads <gage.eads@intel.com>
> ---
>  doc/api/doxy-api-index.md       |   3 +-
>  drivers/event/dlb/dlb.c         | 145 ++++++++++++++++++++++++++++++++++++----
>  drivers/event/dlb/dlb_priv.h    |   3 +
>  drivers/event/dlb/meson.build   |   4 +-
>  drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++
>  drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++
>  drivers/event/dlb/version.map   |   6 ++
>  7 files changed, 262 insertions(+), 14 deletions(-)
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
>
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index a9c12d1..2ff3f74 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -51,7 +51,8 @@ The public API headers are grouped by topics:
>    [dpaa2_mempool]      (@ref rte_dpaa2_mempool.h),
>    [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
>    [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
> -  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
> +  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h),
> +  [dlb]                (@ref rte_pmd_dlb.h),

No comma at the end of the list.

Plus, missing the dox-api.conf.in update.


>
>  - **memory**:
>    [memseg]             (@ref rte_memory.h),


> diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
> index 552ff9d..7f38c30 100644
> --- a/drivers/event/dlb/meson.build
> +++ b/drivers/event/dlb/meson.build
> @@ -12,7 +12,9 @@ sources = files('dlb.c',
>                 'dlb_xstats.c',
>                 'pf/dlb_main.c',
>                 'pf/dlb_pf.c',
> -               'pf/base/dlb_resource.c'
> +               'pf/base/dlb_resource.c',
> +               'rte_pmd_dlb.c',
>  )
>
>  deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
> +install_headers('rte_pmd_dlb.h')

We introduced the headers variable, please use it and do not call
install_headers directly.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 20/23] event/dlb: add PMD's token pop public interface

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 5:09 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>;
> dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Thomas Monjalon
> <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 20/23] event/dlb: add PMD's token pop
> public interface
> 
> On Sat, Oct 31, 2020 at 7:23 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > The PMD uses a public interface to allow applications to
> > control the token pop mode. Supported token pop modes are
> > as follows, and they impact core scheduling affinity for
> > ldb ports.
> >
> > AUTO_POP: Pop the CQ tokens immediately after dequeueing.
> > DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
> >              are released. Supported on load-balanced ports
> >              only.
> > DEFERRED_POP: Pop the CQ tokens during next dequeue operation.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > Reviewed-by: Gage Eads <gage.eads@intel.com>
> > ---
> >  doc/api/doxy-api-index.md       |   3 +-
> >  drivers/event/dlb/dlb.c         | 145
> ++++++++++++++++++++++++++++++++++++----
> >  drivers/event/dlb/dlb_priv.h    |   3 +
> >  drivers/event/dlb/meson.build   |   4 +-
> >  drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++
> >  drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++
> >  drivers/event/dlb/version.map   |   6 ++
> >  7 files changed, 262 insertions(+), 14 deletions(-)
> >  create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
> >  create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
> >
> > diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> > index a9c12d1..2ff3f74 100644
> > --- a/doc/api/doxy-api-index.md
> > +++ b/doc/api/doxy-api-index.md
> > @@ -51,7 +51,8 @@ The public API headers are grouped by topics:
> >    [dpaa2_mempool]      (@ref rte_dpaa2_mempool.h),
> >    [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
> >    [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
> > -  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
> > +  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h),
> > +  [dlb]                (@ref rte_pmd_dlb.h),
> 
> No comma at the end of the list.
> 
> Plus, missing the dox-api.conf.in update.
> 
> 
> >
> >  - **memory**:
> >    [memseg]             (@ref rte_memory.h),
> 
> 
> > diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
> > index 552ff9d..7f38c30 100644
> > --- a/drivers/event/dlb/meson.build
> > +++ b/drivers/event/dlb/meson.build
> > @@ -12,7 +12,9 @@ sources = files('dlb.c',
> >                 'dlb_xstats.c',
> >                 'pf/dlb_main.c',
> >                 'pf/dlb_pf.c',
> > -               'pf/base/dlb_resource.c'
> > +               'pf/base/dlb_resource.c',
> > +               'rte_pmd_dlb.c',
> >  )
> >
> >  deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
> > +install_headers('rte_pmd_dlb.h')
> 
> We introduced the headers variable, please use it and do not call
> install_headers directly.
> 
> 
> --
> David Marchand


Addressed in next patch-set

[dpdk-dev] [PATCH v14 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 12cf38b..07bd4e2 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3828,6 +3828,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..aefadab
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 7f38c30..875cf89 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 21/23] event/dlb: add PMD self-tests

[David Marchand]

On Sat, Oct 31, 2020 at 7:24 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
> +static inline int
> +create_ports(int num_ports)
> +{
> +       int i;
> +
> +       if (num_ports > MAX_PORTS)
> +               return -1;
> +
> +       for (i = 0; i < num_ports; i++) {
> +               struct rte_event_port_conf conf;
> +
> +               if (rte_event_port_default_conf_get(evdev, i, &conf)) {
> +                       printf("%d: Error querying default port conf\n",
> +                              __LINE__);
> +                       return -1;
> +               }
> +
> +               if (rte_event_port_setup(evdev, i, &conf) < 0) {
> +                       printf("%d: Error setting up port %d\n", i, __LINE__);

Caught while diffing the dlb and dlb2 drivers...
Here, it should be __LINE__, i.


> +                       return -1;
> +               }
> +       }
> +
> +       return 0;
> +}


-- 
David Marchand

[dpdk-dev] [PATCH v14 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 07bd4e2..713b1e6 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3804,6 +3807,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3818,7 +3843,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v14 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 713b1e6..0f688bf 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3850,6 +3862,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

Re: [dpdk-dev] [PATCH v14 00/23] Add DLB PMD

[David Marchand]

On Sat, Oct 31, 2020 at 7:16 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> The following patch series adds support for a new eventdev PMD. The DLB
> PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> The DLB is a PCIe device that provides load-balanced, prioritized
> scheduling of core-to-core communication. The device consists of
> queues and arbiters that connect producer and consumer cores, and
> implements load-balanced queueing features including:
> - Lock-free multi-producer/multi-consumer operation.
> - Multiple priority levels for varying traffic types.
> - 'Direct' traffic (i.e. multi-producer/single-consumer)
> - Simple unordered load-balanced distribution.
> - Atomic lock-free load balancing across multiple consumers.
> - Queue element reordering feature allowing ordered load-balanced
>   distribution.
>
> The DLB hardware supports both load balanced and directed ports and
> queues. Unlike other eventdev devices already in the repo,  not all
> DLB ports and queues are equally capable. In particular, directed
> ports are limited to a single link, and must be connected to a directed
> queue.
> Additionally, even though LDB ports may link multiple queues, the
> number of queues that may be linked is limited by hardware. Another
> difference is that DLB does not have a straightforward way of carrying
> the flow_id in the queue elements (QE) that the hardware operates on.
>
> While reviewing the code, please be aware that this PMD has full
> control over the DLB hardware. Intel will be extending the DLB PMD
> in the future (not as part of this first series) with a mode that we
> refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> kernel driver to configure the device, ports, and queues, and memory
> maps device MMIO so datapath operations occur purely in user-space.
>
> The framework to support both the PF PMD and bifurcated PMD exists in
> this patchset, and is why the iface.[ch] layer is present.
>
> Major changes in V14
> ====================
> - Fixed format errors in doc/api/doxy-api-index.md
> - Delayed introduction of dlb2_consume_qe_immediate until
>   add-dequeue-and-its-burst-variants.patch
> - Delayed introduction of dlb2_construct_token_pop_qe until
>   add-PMD-s-token-pop-public-interface.patch
> - Delayed introduction of dlb_equeue_*_delayed until
>   add dequeue and its burst variants.patch

I just sent a bunch of comments.
I still see a build error with clang for unused stuff.

There is no point in sending a new series unless the clang build is
fixed once and for all.


I compared dlb and dlb2 code.
I presume fixing bugs in the future will amount to double patches every time.


But, on a positive note, France won against Ireland.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v14 00/23] Add DLB PMD

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, October 31, 2020 5:16 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v14 00/23] Add DLB PMD
> 
> On Sat, Oct 31, 2020 at 7:16 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > The following patch series adds support for a new eventdev PMD. The DLB
> > PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> > The DLB is a PCIe device that provides load-balanced, prioritized
> > scheduling of core-to-core communication. The device consists of
> > queues and arbiters that connect producer and consumer cores, and
> > implements load-balanced queueing features including:
> > - Lock-free multi-producer/multi-consumer operation.
> > - Multiple priority levels for varying traffic types.
> > - 'Direct' traffic (i.e. multi-producer/single-consumer)
> > - Simple unordered load-balanced distribution.
> > - Atomic lock-free load balancing across multiple consumers.
> > - Queue element reordering feature allowing ordered load-balanced
> >   distribution.
> >
> > The DLB hardware supports both load balanced and directed ports and
> > queues. Unlike other eventdev devices already in the repo,  not all
> > DLB ports and queues are equally capable. In particular, directed
> > ports are limited to a single link, and must be connected to a directed
> > queue.
> > Additionally, even though LDB ports may link multiple queues, the
> > number of queues that may be linked is limited by hardware. Another
> > difference is that DLB does not have a straightforward way of carrying
> > the flow_id in the queue elements (QE) that the hardware operates on.
> >
> > While reviewing the code, please be aware that this PMD has full
> > control over the DLB hardware. Intel will be extending the DLB PMD
> > in the future (not as part of this first series) with a mode that we
> > refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> > kernel driver to configure the device, ports, and queues, and memory
> > maps device MMIO so datapath operations occur purely in user-space.
> >
> > The framework to support both the PF PMD and bifurcated PMD exists in
> > this patchset, and is why the iface.[ch] layer is present.
> >
> > Major changes in V14
> > ====================
> > - Fixed format errors in doc/api/doxy-api-index.md
> > - Delayed introduction of dlb2_consume_qe_immediate until
> >   add-dequeue-and-its-burst-variants.patch
> > - Delayed introduction of dlb2_construct_token_pop_qe until
> >   add-PMD-s-token-pop-public-interface.patch
> > - Delayed introduction of dlb_equeue_*_delayed until
> >   add dequeue and its burst variants.patch
> 
> I just sent a bunch of comments.
> I still see a build error with clang for unused stuff.
> 
> There is no point in sending a new series unless the clang build is
> fixed once and for all.
> 
> 
> I compared dlb and dlb2 code.
> I presume fixing bugs in the future will amount to double patches every time.
> 
> 
> But, on a positive note, France won against Ireland.
> 
> 
> --
> David Marchand

Where do I find the clang output? I followed the links in the 0-day email, and from patchwork clicking on the patches with the red failure indicator,
but none of those seemed to lead me to any clang error output. My build server does not have clang, which presents a problem
for me.  I've tried to use gcc -Wunused and it failed to catch anything. Is there a way for me to submit a clang job to a dpdk server.

Thanks,
Tim

Re: [dpdk-dev] [PATCH v14 00/23] Add DLB PMD

[David Marchand]

On Sat, Oct 31, 2020 at 11:25 PM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
> > I still see a build error with clang for unused stuff.
> >
> > There is no point in sending a new series unless the clang build is
> > fixed once and for all.
> >
> Where do I find the clang output? I followed the links in the 0-day email, and from patchwork clicking on the patches with the red failure indicator,
> but none of those seemed to lead me to any clang error output. My build server does not have clang, which presents a problem
> for me.  I've tried to use gcc -Wunused and it failed to catch anything. Is there a way for me to submit a clang job to a dpdk server.

I did not think that installing clang was such a hurdle...

You could check this with per patch compilation in Travis but that
would be a waste of cpu/time.

I guess the reason why gcc does not complain is because those symbols
are marked with inline.

"""
       -Wunused-function
           Warn whenever a static function is declared but not defined
or a non-inline static function is unused.  This warning is enabled by
-Wall.
"""

Please remove "inline" in all .c unless you have numbers showing it
has a beneficial impact.

Looking with the whole patchset applied:
$ git grep -w inline 'drivers/event/dlb/**.c'
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline void
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline void
drivers/event/dlb/dlb.c:static inline void
drivers/event/dlb/dlb.c:static inline void
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline uint16_t
drivers/event/dlb/dlb.c:static inline void
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline int
drivers/event/dlb/dlb.c:static inline int16_t
drivers/event/dlb/dlb.c:static inline int16_t
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/dlb_selftest.c:static inline int
drivers/event/dlb/pf/base/dlb_resource.c:static inline void
dlb_flush_csr(struct dlb_hw *hw)


Stopping at 84be92444bb9 ("event/dlb: add dequeue and its burst
variants"), then removing those inline tokens, the problem is
reproduced even with gcc:

FAILED: drivers/a715181@@tmp_rte_event_dlb@sta/event_dlb_dlb.c.o
ccache gcc -Idrivers/a715181@@tmp_rte_event_dlb@sta -Idrivers
-I../../dpdk/drivers -Idrivers/event/dlb
-I../../dpdk/drivers/event/dlb -Ilib/librte_eventdev
-I../../dpdk/lib/librte_eventdev -I. -I../../dpdk/ -Iconfig
-I../../dpdk/config -Ilib/librte_eal/include
-I../../dpdk/lib/librte_eal/include -Ilib/librte_eal/linux/include
-I../../dpdk/lib/librte_eal/linux/include -Ilib/librte_eal/x86/include
-I../../dpdk/lib/librte_eal/x86/include -Ilib/librte_eal/common
-I../../dpdk/lib/librte_eal/common -Ilib/librte_eal
-I../../dpdk/lib/librte_eal -Ilib/librte_kvargs
-I../../dpdk/lib/librte_kvargs
-Ilib/librte_telemetry/../librte_metrics
-I../../dpdk/lib/librte_telemetry/../librte_metrics
-Ilib/librte_telemetry -I../../dpdk/lib/librte_telemetry
-Ilib/librte_ring -I../../dpdk/lib/librte_ring -Ilib/librte_ethdev
-I../../dpdk/lib/librte_ethdev -Ilib/librte_net
-I../../dpdk/lib/librte_net -Ilib/librte_mbuf
-I../../dpdk/lib/librte_mbuf -Ilib/librte_mempool
-I../../dpdk/lib/librte_mempool -Ilib/librte_meter
-I../../dpdk/lib/librte_meter -Ilib/librte_hash
-I../../dpdk/lib/librte_hash -Ilib/librte_rcu
-I../../dpdk/lib/librte_rcu -Ilib/librte_timer
-I../../dpdk/lib/librte_timer -Ilib/librte_cryptodev
-I../../dpdk/lib/librte_cryptodev -Ilib/librte_pci
-I../../dpdk/lib/librte_pci -Idrivers/bus/pci
-I../../dpdk/drivers/bus/pci -I../../dpdk/drivers/bus/pci/linux
-I/home/dmarchan/intel-ipsec-mb/install/include
-fdiagnostics-color=always -pipe -D_FILE_OFFSET_BITS=64 -Wall
-Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual
-Wdeprecated -Wformat-nonliteral -Wformat-security
-Wmissing-declarations -Wmissing-prototypes -Wnested-externs
-Wold-style-definition -Wpointer-arith -Wsign-compare
-Wstrict-prototypes -Wundef -Wwrite-strings
-Wno-address-of-packed-member -Wno-packed-not-aligned
-Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native
-DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -Wno-format-truncation
-MD -MQ 'drivers/a715181@@tmp_rte_event_dlb@sta/event_dlb_dlb.c.o' -MF
'drivers/a715181@@tmp_rte_event_dlb@sta/event_dlb_dlb.c.o.d' -o
'drivers/a715181@@tmp_rte_event_dlb@sta/event_dlb_dlb.c.o' -c
../../dpdk/drivers/event/dlb/dlb.c
../../dpdk/drivers/event/dlb/dlb.c:2784:1: error:
‘dlb_event_enqueue_forward_burst_delayed’ defined but not used
[-Werror=unused-function]
 2784 | dlb_event_enqueue_forward_burst_delayed(void *event_port,
      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../../dpdk/drivers/event/dlb/dlb.c:2768:1: error:
‘dlb_event_enqueue_new_burst_delayed’ defined but not used
[-Werror=unused-function]
 2768 | dlb_event_enqueue_new_burst_delayed(void *event_port,
      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
../../dpdk/drivers/event/dlb/dlb.c:2753:1: error:
‘dlb_event_enqueue_delayed’ defined but not used
[-Werror=unused-function]
 2753 | dlb_event_enqueue_delayed(void *event_port,
      | ^~~~~~~~~~~~~~~~~~~~~~~~~
../../dpdk/drivers/event/dlb/dlb.c:2738:1: error:
‘dlb_event_enqueue_burst_delayed’ defined but not used
[-Werror=unused-function]
 2738 | dlb_event_enqueue_burst_delayed(void *event_port,
      | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
ninja: build stopped: subcommand failed.


-- 
David Marchand

[dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V15
====================
Address comments from David Marchand:
- this patch-set is based on Nov 1, 2020 dpdk-next-eventdev
- fix docs build (doxy-api.conf.in and doxy-api-index.md)
- restore blank line in MAINTAINERS file
- move dlb announcement in release_20_11.rst after ethdev
- use headers = files() for exported meson public headers
- fix a typo in 'add documentation ..." commit message
- use eal version of cldemote
- convert a couple of printfs to LOG messages
- fix missing "~" in dlb documentation
- delay introduction of _delayed token pop functions to
  token pop commit (fixes 8 or so unused function errors)
- all patches build incrementally (gcc), and checkpatches reports
  success
- I am not able to run clang locally. If clang errors are still
  present I will ask IT to install clang on a build server tomorrow.

Major changes in V14
====================
- Fixed format errors in doc/api/doxy-api-index.md
- Delayed introduction of dlb2_consume_qe_immediate until
  add-dequeue-and-its-burst-variants.patch
- Delayed introduction of dlb2_construct_token_pop_qe until 
  add-PMD-s-token-pop-public-interface.patch
- Delayed introduction of dlb_equeue_*_delayed until
  add dequeue and its burst variants.patch

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with
  _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    5 +
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    3 +-
 doc/api/doxy-api.conf.in                     |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4079 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   36 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1217 +++++
 drivers/event/dlb/meson.build                |   22 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 33 files changed, 21677 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v15 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availability of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 MAINTAINERS                            |  5 +++++
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 15 ++++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5b390d1..9600d7a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1176,6 +1176,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index 88b9086..6c33dbe 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -295,6 +295,11 @@ New Features
   Added performance tuning arguments to allow tuning the scheduler for
   better throughtput in high core count use cases.
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Updated ioat rawdev driver**
 
   The ioat rawdev driver has been updated and enhanced. Changes include:
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..68cb0ae
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+headers = files()
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v15 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 68cb0ae..50209bf 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,8 +7,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 headers = files()
 
-- 
2.6.4

[dpdk-dev] [PATCH v15 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v15 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v15 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..aae94dc
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_INLINE_FNS_H_
+#define _DLB_INLINE_FNS_H_
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		:
+		: "a" (dest), "d" (src));
+}
+
+#endif /* _DLB_INLINE_FNS_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v15 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and
will be introduced in subsequent patches contained in
this patch-set.

Hardware init has been moved to a subsequent patch in order to
minimize the patch size.

Initialization of the flexible interface layer has been moved to
a subsequent patch in order to minimize patch size.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 50209bf..61c0182 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,10 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 headers = files()
 
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..00ab732
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..c10c36c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v15 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 61c0182..0e66cdc 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v15 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 0e66cdc..5502647 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 headers = files()
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index c10c36c..2f4a828 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v15 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1217 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1241 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..89186d5
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1217 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		DLB_LOG_ERR("Invalid file pointer\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5502647..66a3fbe 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v15 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v15 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v15 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..2b050ad 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		DLB_LOG_ERR("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v15 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..bb3455b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2b050ad..09e4640 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v15 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 09e4640..29d5a0c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v15 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 29d5a0c..748789c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v15 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 748789c..ea57c28 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v15 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 161 ++++++++++++
 drivers/event/dlb/dlb.c      | 567 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 728 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index bb3455b..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index ea57c28..4053679 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2135,6 +2135,568 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2721,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v15 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 719 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 740 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4053679..1aab7ce 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2605,6 +2606,46 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,
 	dlb_pp_write(qm_port->qe4, port_data);
 }
 
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
@@ -2697,9 +2738,678 @@ dlb_event_enqueue_forward_burst(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	rte_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2726,6 +3436,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v15 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1aab7ce..95b7bbe 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -72,17 +72,6 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3405,6 +3394,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3414,6 +3646,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v15 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   3 +-
 doc/api/doxy-api.conf.in        |   1 +
 drivers/event/dlb/dlb.c         | 187 ++++++++++++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   5 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 ++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 8 files changed, 309 insertions(+), 11 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index b865a51..de2d113 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,7 +52,8 @@ The public API headers are grouped by topics:
   [dpaa2_mempool]      (@ref rte_dpaa2_mempool.h),
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
-  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h),
+  [dlb]                (@ref rte_pmd_dlb.h)
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index c5b01a1..116acf6 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -7,6 +7,7 @@ USE_MDFILE_AS_MAINPAGE  = @TOPDIR@/doc/api/doxy-api-index.md
 INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/bus/vdev \
                           @TOPDIR@/drivers/crypto/scheduler \
+                          @TOPDIR@/drivers/event/dlb \
                           @TOPDIR@/drivers/mempool/dpaa2 \
                           @TOPDIR@/drivers/net/ark \
                           @TOPDIR@/drivers/net/bnxt \
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 95b7bbe..e59a0de 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -72,6 +72,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -1003,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1163,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2572,6 +2620,30 @@ dlb_event_build_hcws(struct dlb_port *qm_port,
 	}
 }
 
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+	qm_port->owed_tokens = 0;
+}
+
 static __rte_always_inline void
 dlb_pp_write(struct dlb_enqueue_qe *qe4,
 	     struct process_local_port_data *port_data)
@@ -2638,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2666,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2701,14 +2803,29 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2716,7 +2833,15 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2724,7 +2849,15 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3124,7 +3257,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3249,7 +3383,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3293,6 +3428,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3325,6 +3482,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3344,6 +3502,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3362,6 +3524,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3381,6 +3544,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3687,7 +3854,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3736,6 +3903,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 66a3fbe..2ea3b4c 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,9 +12,10 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
-headers = files()
+headers = files('rte_pmd_dlb.h')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v15 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e59a0de..afef56c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3828,6 +3828,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..b9ef778
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 2ea3b4c..7d40c16 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 headers = files('rte_pmd_dlb.h')
-- 
2.6.4

[dpdk-dev] [PATCH v15 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index afef56c..bda696a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3804,6 +3807,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3818,7 +3843,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v15 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index bda696a..2bb270d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3850,6 +3862,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[David Marchand]

On Sun, Nov 1, 2020 at 8:25 PM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> The following patch series adds support for a new eventdev PMD. The DLB
> PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> The DLB is a PCIe device that provides load-balanced, prioritized
> scheduling of core-to-core communication. The device consists of
> queues and arbiters that connect producer and consumer cores, and
> implements load-balanced queueing features including:
> - Lock-free multi-producer/multi-consumer operation.
> - Multiple priority levels for varying traffic types.
> - 'Direct' traffic (i.e. multi-producer/single-consumer)
> - Simple unordered load-balanced distribution.
> - Atomic lock-free load balancing across multiple consumers.
> - Queue element reordering feature allowing ordered load-balanced
>   distribution.
>
> The DLB hardware supports both load balanced and directed ports and
> queues. Unlike other eventdev devices already in the repo,  not all
> DLB ports and queues are equally capable. In particular, directed
> ports are limited to a single link, and must be connected to a directed
> queue.
> Additionally, even though LDB ports may link multiple queues, the
> number of queues that may be linked is limited by hardware. Another
> difference is that DLB does not have a straightforward way of carrying
> the flow_id in the queue elements (QE) that the hardware operates on.
>
> While reviewing the code, please be aware that this PMD has full
> control over the DLB hardware. Intel will be extending the DLB PMD
> in the future (not as part of this first series) with a mode that we
> refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> kernel driver to configure the device, ports, and queues, and memory
> maps device MMIO so datapath operations occur purely in user-space.
>
> The framework to support both the PF PMD and bifurcated PMD exists in
> this patchset, and is why the iface.[ch] layer is present.
>
> Major changes in V15
> ====================
> Address comments from David Marchand:
> - this patch-set is based on Nov 1, 2020 dpdk-next-eventdev
> - fix docs build (doxy-api.conf.in and doxy-api-index.md)
> - restore blank line in MAINTAINERS file
> - move dlb announcement in release_20_11.rst after ethdev
> - use headers = files() for exported meson public headers
> - fix a typo in 'add documentation ..." commit message
> - use eal version of cldemote
> - convert a couple of printfs to LOG messages

There are still printf in drivers/event/dlb/pf/dlb_main.c from a grep
I did on v15.
Why were they kept?

+ I did not look too much at the PCI code in this file, but I suspect
we could factor some of it with librte_pci.


> - fix missing "~" in dlb documentation
> - delay introduction of _delayed token pop functions to
>   token pop commit (fixes 8 or so unused function errors)
> - all patches build incrementally (gcc), and checkpatches reports
>   success

There is a valid warning on patch 6:
http://mails.dpdk.org/archives/test-report/2020-November/164347.html

WARNING:REPEATED_WORD: Possible repeated word: 'of'
#1493: FILE: drivers/event/dlb/pf/base/dlb_osdep_bitmap.h:364:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,


> - I am not able to run clang locally. If clang errors are still
>   present I will ask IT to install clang on a build server tomorrow.

This time, compilation passed for each patch on my build system (using
./devtools/test-meson-builds.sh, x86, arm, ppc + doc generation).


-- 
David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Sunday, November 1, 2020 3:11 PM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>
> Subject: Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD
> 
> On Sun, Nov 1, 2020 at 8:25 PM Timothy McDaniel
> <timothy.mcdaniel@intel.com> wrote:
> >
> > The following patch series adds support for a new eventdev PMD. The DLB
> > PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> > The DLB is a PCIe device that provides load-balanced, prioritized
> > scheduling of core-to-core communication. The device consists of
> > queues and arbiters that connect producer and consumer cores, and
> > implements load-balanced queueing features including:
> > - Lock-free multi-producer/multi-consumer operation.
> > - Multiple priority levels for varying traffic types.
> > - 'Direct' traffic (i.e. multi-producer/single-consumer)
> > - Simple unordered load-balanced distribution.
> > - Atomic lock-free load balancing across multiple consumers.
> > - Queue element reordering feature allowing ordered load-balanced
> >   distribution.
> >
> > The DLB hardware supports both load balanced and directed ports and
> > queues. Unlike other eventdev devices already in the repo,  not all
> > DLB ports and queues are equally capable. In particular, directed
> > ports are limited to a single link, and must be connected to a directed
> > queue.
> > Additionally, even though LDB ports may link multiple queues, the
> > number of queues that may be linked is limited by hardware. Another
> > difference is that DLB does not have a straightforward way of carrying
> > the flow_id in the queue elements (QE) that the hardware operates on.
> >
> > While reviewing the code, please be aware that this PMD has full
> > control over the DLB hardware. Intel will be extending the DLB PMD
> > in the future (not as part of this first series) with a mode that we
> > refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> > kernel driver to configure the device, ports, and queues, and memory
> > maps device MMIO so datapath operations occur purely in user-space.
> >
> > The framework to support both the PF PMD and bifurcated PMD exists in
> > this patchset, and is why the iface.[ch] layer is present.
> >
> > Major changes in V15
> > ====================
> > Address comments from David Marchand:
> > - this patch-set is based on Nov 1, 2020 dpdk-next-eventdev
> > - fix docs build (doxy-api.conf.in and doxy-api-index.md)
> > - restore blank line in MAINTAINERS file
> > - move dlb announcement in release_20_11.rst after ethdev
> > - use headers = files() for exported meson public headers
> > - fix a typo in 'add documentation ..." commit message
> > - use eal version of cldemote
> > - convert a couple of printfs to LOG messages
> 
> There are still printf in drivers/event/dlb/pf/dlb_main.c from a grep
> I did on v15.
> Why were they kept?
> 

Sorry.  This was an oversight on my part. I thought you had identified all of the files with printf.
I should have searched all of the files, and fixed any cases that were not identified in the CR.
I will fix these in the next patch-set.

> + I did not look too much at the PCI code in this file, but I suspect
> we could factor some of it with librte_pci.
> 

Can we look at possibly doing that post rc2 merge?

> 
> > - fix missing "~" in dlb documentation
> > - delay introduction of _delayed token pop functions to
> >   token pop commit (fixes 8 or so unused function errors)
> > - all patches build incrementally (gcc), and checkpatches reports
> >   success
> 
> There is a valid warning on patch 6:
> http://mails.dpdk.org/archives/test-report/2020-November/164347.html
> 
> WARNING:REPEATED_WORD: Possible repeated word: 'of'
> #1493: FILE: drivers/event/dlb/pf/base/dlb_osdep_bitmap.h:364:
> + * Returns the bitmap's longest contiguous range of of set bits upon success,
> 

I will address with the next patch-set. Odd that I did not see the error/warning when I
ran devtools/checkpatches.sh

> 
> > - I am not able to run clang locally. If clang errors are still
> >   present I will ask IT to install clang on a build server tomorrow.
> 
> This time, compilation passed for each patch on my build system (using
> ./devtools/test-meson-builds.sh, x86, arm, ppc + doc generation).

Excellent news. Thanks for the update.

> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[David Marchand]

On Sun, Nov 1, 2020 at 10:26 PM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
> > + I did not look too much at the PCI code in this file, but I suspect
> > we could factor some of it with librte_pci.
> >
>
> Can we look at possibly doing that post rc2 merge?

Please try to look at dpdk existing API before adding code.
Ok for a followup cleanup.


>
> >
> > > - fix missing "~" in dlb documentation
> > > - delay introduction of _delayed token pop functions to
> > >   token pop commit (fixes 8 or so unused function errors)
> > > - all patches build incrementally (gcc), and checkpatches reports
> > >   success
> >
> > There is a valid warning on patch 6:
> > http://mails.dpdk.org/archives/test-report/2020-November/164347.html
> >
> > WARNING:REPEATED_WORD: Possible repeated word: 'of'
> > #1493: FILE: drivers/event/dlb/pf/base/dlb_osdep_bitmap.h:364:
> > + * Returns the bitmap's longest contiguous range of of set bits upon success,
> >
>
> I will address with the next patch-set. Odd that I did not see the error/warning when I
> ran devtools/checkpatches.sh

You need a fairly recent checkpatch.pl.



-- 
David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[David Marchand]

On Sun, Nov 1, 2020 at 10:26 PM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
> > + I did not look too much at the PCI code in this file, but I suspect
> > we could factor some of it with librte_pci.
> >
>
> Can we look at possibly doing that post rc2 merge?

Any update?


-- 
David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[McDaniel, Timothy]

I'm not seeing where we can use librte_pci here.  We are already using the dpdk low level rte_pci_read_config(..) APIs.
Am I missing something?

Thanks,
Tim


> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Tuesday, November 10, 2020 6:52 AM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>; Gaetan
> Rivet <grive@u256.net>
> Subject: Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD
> 
> On Sun, Nov 1, 2020 at 10:26 PM McDaniel, Timothy
> <timothy.mcdaniel@intel.com> wrote:
> > > + I did not look too much at the PCI code in this file, but I suspect
> > > we could factor some of it with librte_pci.
> > >
> >
> > Can we look at possibly doing that post rc2 merge?
> 
> Any update?
> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[David Marchand]

On Tue, Nov 10, 2020 at 4:29 PM McDaniel, Timothy
<timothy.mcdaniel@intel.com> wrote:
>
> I'm not seeing where we can use librte_pci here.  We are already using the dpdk low level rte_pci_read_config(..) APIs.
> Am I missing something?

dlb_pci_find_ext_capability is a copy of rte_pci_find_ext_capability
from the pci bus.

A lot of DLB_PCI_* defines have a librte_pci equivalent.
Plus, if there are generic pci defines needed by DLB, they would
rather belong to librte_pci than a driver.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Tuesday, November 10, 2020 9:51 AM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>; Gaetan
> Rivet <grive@u256.net>
> Subject: Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD
> 
> On Tue, Nov 10, 2020 at 4:29 PM McDaniel, Timothy
> <timothy.mcdaniel@intel.com> wrote:
> >
> > I'm not seeing where we can use librte_pci here.  We are already using the
> dpdk low level rte_pci_read_config(..) APIs.
> > Am I missing something?
> 
> dlb_pci_find_ext_capability is a copy of rte_pci_find_ext_capability
> from the pci bus.
> 
> A lot of DLB_PCI_* defines have a librte_pci equivalent.
> Plus, if there are generic pci defines needed by DLB, they would
> rather belong to librte_pci than a driver.
> 
> 
> --
> David Marchand

Thanks, David.  I missed that. I'll get to work on removing the duplication.

Tim

Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD

[McDaniel, Timothy]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Tuesday, November 10, 2020 9:51 AM
> To: McDaniel, Timothy <timothy.mcdaniel@intel.com>
> Cc: dev <dev@dpdk.org>; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads,
> Gage <gage.eads@intel.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Thomas Monjalon <thomas@monjalon.net>; Gaetan
> Rivet <grive@u256.net>
> Subject: Re: [dpdk-dev] [PATCH v15 00/23] Add DLB PMD
> 
> On Tue, Nov 10, 2020 at 4:29 PM McDaniel, Timothy
> <timothy.mcdaniel@intel.com> wrote:
> >
> > I'm not seeing where we can use librte_pci here.  We are already using the
> dpdk low level rte_pci_read_config(..) APIs.
> > Am I missing something?
> 
> dlb_pci_find_ext_capability is a copy of rte_pci_find_ext_capability
> from the pci bus.
> 
> A lot of DLB_PCI_* defines have a librte_pci equivalent.
> Plus, if there are generic pci defines needed by DLB, they would
> rather belong to librte_pci than a driver.
> 
> 
> --
> David Marchand

David,

I just uploaded patches that should address most of your concerns.
See https://patches.dpdk.org/patch/84034/ and https://patches.dpdk.org/patch/84031/ 

Thanks,
Tim

[dpdk-dev] [PATCH v16 00/23] Add DLB PMD

[Timothy McDaniel]

The following patch series adds support for a new eventdev PMD. The DLB
PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
The DLB is a PCIe device that provides load-balanced, prioritized
scheduling of core-to-core communication. The device consists of
queues and arbiters that connect producer and consumer cores, and
implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced
  distribution.

The DLB hardware supports both load balanced and directed ports and
queues. Unlike other eventdev devices already in the repo,  not all
DLB ports and queues are equally capable. In particular, directed
ports are limited to a single link, and must be connected to a directed
queue.
Additionally, even though LDB ports may link multiple queues, the
number of queues that may be linked is limited by hardware. Another
difference is that DLB does not have a straightforward way of carrying
the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full
control over the DLB hardware. Intel will be extending the DLB PMD
in the future (not as part of this first series) with a mode that we
refer to as the bifurcated PMD. The bifurcated PMD communicates with a
kernel driver to configure the device, ports, and queues, and memory
maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in
this patchset, and is why the iface.[ch] layer is present.

Major changes in V16
====================
Address additional comments from David Marchand:
- converted printfs in dlb/pf/dlb_main.c to DLB_LOG
- fixed a repeated word error in dlb/pf/base/osdep_bitmap.h
- caught up with marking the patches that Gage reviewed

Major changes in V15
====================
Address comments from David Marchand:
- this patch-set is based on Nov 1, 2020 dpdk-next-eventdev
- fix docs build (doxy-api.conf.in and doxy-api-index.md)
- restore blank line in MAINTAINERS file
- move dlb announcement in release_20_11.rst after ethdev
- use headers = files() for exported meson public headers
- fix a typo in 'add documentation ..." commit message
- use eal version of cldemote
- convert a couple of printfs to LOG messages
- fix missing "~" in dlb documentation
- delay introduction of _delayed token pop functions to
  token pop commit (fixes 8 or so unused function errors)
- all patches build incrementally (gcc), and checkpatches reports
  success
- I am not able to run clang locally. If clang errors are still
  present I will ask IT to install clang on a build server tomorrow.

Major changes in V14
====================
- Fixed format errors in doc/api/doxy-api-index.md
- Delayed introduction of dlb2_consume_qe_immediate until
  add-dequeue-and-its-burst-variants.patch
- Delayed introduction of dlb2_construct_token_pop_qe until 
  add-PMD-s-token-pop-public-interface.patch
- Delayed introduction of dlb_equeue_*_delayed until
  add dequeue and its burst variants.patch

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with
  _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after
  the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of
  umonitor/umwait, rather than the new functions in the power
  patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build
  file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The
  software is very similar in form so some DLB2 reviews comments
  were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews
by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI
  named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ
memory for depths < 8. Hardware requires minimum allocation to be
at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias.
  I will address the remaining items identified by Mattias in the next
  patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
  event/dlb: add documentation and meson infrastructure
  event/dlb: add dynamic logging
  event/dlb: add private data structures and constants
  event/dlb: add definitions shared with LKM or shared code
  event/dlb: add inline functions
  event/dlb: add eventdev probe
  event/dlb: add flexible interface
  event/dlb: add probe-time hardware init
  event/dlb: add xstats
  event/dlb: add infos get and configure
  event/dlb: add queue and port default conf
  event/dlb: add queue setup
  event/dlb: add port setup
  event/dlb: add port link
  event/dlb: add port unlink and port unlinks in progress
  event/dlb: add eventdev start
  event/dlb: add enqueue and its burst variants
  event/dlb: add dequeue and its burst variants
  event/dlb: add eventdev stop and close
  event/dlb: add PMD's token pop public interface
  event/dlb: add PMD self-tests
  event/dlb: add queue and port release
  event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    5 +
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    3 +-
 doc/api/doxy-api.conf.in                     |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4079 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   36 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1217 +++++
 drivers/event/dlb/meson.build                |   22 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 33 files changed, 21677 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

-- 
2.6.4

[dpdk-dev] [PATCH v16 01/23] event/dlb: add documentation and meson infrastructure

[Timothy McDaniel]

Note that config/rte_config.h contains several configuration
switches, providing for fine control of the PMD's
runtime behaviour.

The meson infrastructure is expanded as additional files are
added to this patchset.

Adds announcement of availability of the new driver
for Intel Dynamic Load Balancer 1.0 hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 MAINTAINERS                            |  5 +++++
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 15 ++++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5b390d1..9600d7a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1176,6 +1176,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <timothy.mcdaniel@intel.com>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <pbhagavatula@marvell.com>
 M: Jerin Jacob <jerinj@marvell.com>
diff --git a/config/rte_config.h b/config/rte_config.h
index b78c6aa..9ebe4cc 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 0000000..92341c0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
diff --git a/doc/guides/eventdevs/index.rst b/doc/guides/eventdevs/index.rst
index bb66a5e..4b915bf 100644
--- a/doc/guides/eventdevs/index.rst
+++ b/doc/guides/eventdevs/index.rst
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst
index 88b9086..6c33dbe 100644
--- a/doc/guides/rel_notes/release_20_11.rst
+++ b/doc/guides/rel_notes/release_20_11.rst
@@ -295,6 +295,11 @@ New Features
   Added performance tuning arguments to allow tuning the scheduler for
   better throughtput in high core count use cases.
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Updated ioat rawdev driver**
 
   The ioat rawdev driver has been updated and enhanced. Changes include:
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 0000000..68cb0ae
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+headers = files()
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/event/dlb/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index a7dac99..6601e62 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
-- 
2.6.4

[dpdk-dev] [PATCH v16 02/23] event/dlb: add dynamic logging

[Timothy McDaniel]

This commit adds base support for dynamic logging.
The default log level is NOTICE. Dynamic logging
is used exclusively throughout this patchset.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..e03aa21
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 0000000..c69c9e5
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 68cb0ae..50209bf 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,8 +7,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 headers = files()
 
-- 
2.6.4

[dpdk-dev] [PATCH v16 03/23] event/dlb: add private data structures and constants

[Timothy McDaniel]

Add headers used internally by the PMD.  They include constants,
macros for device resources, structure definitions for hardware interfaces
and software state, and various forward-declarations.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 0000000..f9ff0a5
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v16 04/23] event/dlb: add definitions shared with LKM or shared code

[Timothy McDaniel]

Add headers containing structs and constants shared between
the PMD and the shared code.  The term shared code refers to
the code that implements the hardware interface. The shared code
is introduced in the probe patch, and then is extended as
additional eventdev PMD entry points are added to the patchset.
In the case of the bifurcated PMD (to be introduced in the
future), the shared code is contained in the Linux kernel
module itself.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 0000000..2d9582b
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.6.4

[dpdk-dev] [PATCH v16 05/23] event/dlb: add inline functions

[Timothy McDaniel]

Add miscellaneous inline functions that may be called
from multiple files.  These functions include inline
assembly of new x86 instructions, such as movdir64b,
since they are not available as builtin functions in
the minimum supported GCC version.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 0000000..aae94dc
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_INLINE_FNS_H_
+#define _DLB_INLINE_FNS_H_
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		:
+		: "a" (dest), "d" (src));
+}
+
+#endif /* _DLB_INLINE_FNS_H_ */
-- 
2.6.4

[dpdk-dev] [PATCH v16 06/23] event/dlb: add eventdev probe

[Timothy McDaniel]

Add the eventdev portion of probe, and parse command line
options, but do not initialize hardware.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e03aa21..1659f93 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index f9ff0a5..adb1f7a 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 50209bf..61c0182 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -7,7 +7,10 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 headers = files()
 
diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..4c40e21
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "../../dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+#define DLB_INT_ALARM 0
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	u32 id;
+	u32 domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	u32 id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports;
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a domain is created or destroyed, or
+ * when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..0c119b7
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_log.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+/* Map to PMDs logging interface */
+#define DLB_ERR(dev, fmt, args...) \
+	DLB_LOG_ERR(fmt, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	DLB_LOG_INFO(fmt, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	DLB_LOG_DEBUG(fmt, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function does not deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = rte_ctrl_thread_create(&complete_queue_map_unmap_thread,
+				     "dlb_queue_unmap_waiter",
+				     NULL,
+				     dlb_complete_queue_map_unmap,
+				     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map/unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	/* Not called for PF PMD */
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..4c10c8c
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,441 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (bitmap == NULL || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (bm == NULL)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (mem == NULL) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (bm->map == NULL) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (bitmap == NULL)
+		return;
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_zero() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (bitmap  == NULL || bitmap->map == NULL)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (bitmap  == NULL || bitmap->map  == NULL || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (bitmap == NULL)
+		return -EINVAL;
+
+	if (bitmap->map == NULL)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (dest  == NULL || dest->map == NULL ||
+	    src1 == NULL || src1->map == NULL ||
+	    src2  == NULL || src2->map == NULL)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..a1c63f3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2368 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_SYS_ALARM_INT_ENABLE 0xc001048
+#define DLB_SYS_SYS_ALARM_INT_ENABLE_RST 0x7fffff
+union dlb_sys_sys_alarm_int_enable {
+	struct {
+		u32 cq_addr_overflow_error : 1;
+		u32 ingress_perr : 1;
+		u32 egress_perr : 1;
+		u32 alarm_perr : 1;
+		u32 vf_to_pf_isr_pend_error : 1;
+		u32 pf_to_vf_isr_pend_error : 1;
+		u32 timeout_error : 1;
+		u32 dmvw_sm_error : 1;
+		u32 pptr_sm_par_error : 1;
+		u32 pptr_sm_len_error : 1;
+		u32 sch_sm_error : 1;
+		u32 wbuf_flag_error : 1;
+		u32 dmvw_cl_error : 1;
+		u32 dmvr_cl_error : 1;
+		u32 cmpl_data_error : 1;
+		u32 cmpl_error : 1;
+		u32 fifo_underflow : 1;
+		u32 fifo_overflow : 1;
+		u32 sb_ep_parity_err : 1;
+		u32 ti_parity_err : 1;
+		u32 ri_parity_err : 1;
+		u32 cfgm_ppw_err : 1;
+		u32 system_csr_perr : 1;
+		u32 rsvd0 : 9;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..4f48b73
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,876 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw, int port_id, bool is_ldb);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: port ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of available resources for the PF.
+ */
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ *
+ * This function returns the number of resources in use by the PF. It fills in
+ * the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ */
+void dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				   struct dlb_get_num_resources_args *arg);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pf_to_vf_isr_pend_err() - disable alarm triggered by PF
+ *	access to VF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_vf_to_pf_isr_pend_err() - disable alarm triggered by VF
+ *	access to PF's ISR pending register
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 0000000..f816acb
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,568 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "../dlb_user.h"
+#include "dlb_main.h"
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		DLB_LOG_ERR("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		DLB_LOG_ERR("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		DLB_LOG_ERR("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (dlb_dev == NULL) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+init_driver_state_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int msix_cap_offset, err_cap_offset, acs_cap_offset, wait_count;
+	uint16_t dev_ctl_word, dev_ctl2_word, lnk_word, lnk_word2;
+	uint16_t rt_ctl_word, pri_reqs_dword,  pri_ctrl_word;
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+	uint16_t devsta_busy_word, devctl_word;
+	int pcie_cap_offset, pri_cap_offset;
+	uint16_t slt_word, slt_word2, cmd;
+	int ret = 0, i = 0;
+	uint32_t dword[16];
+	off_t off;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		DLB_LOG_ERR("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			DLB_LOG_ERR("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		DLB_LOG_ERR("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		DLB_LOG_ERR("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		DLB_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				DLB_LOG_ERR("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				DLB_LOG_ERR("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			DLB_LOG_ERR("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	/* Initialize software state */
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	RTE_SET_USED(dlb_dev);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 0000000..22e2152
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 0000000..3f836f3
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 EVDEV_DLB_NAME_PMD_STR,
+						 &dlb_args);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   EVDEV_DLB_NAME_PMD_STR);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		EVDEV_DLB_NAME_PMD_STR);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.6.4

[dpdk-dev] [PATCH v16 07/23] event/dlb: add flexible interface

[Timothy McDaniel]

This commit introduces the flexible interface. This
interface allows the core code to operate in PF mode (direct
hardware access) or bifurcated mode (hardware configured via
kernel driver). This driver currently only supports PF modei,
but bifurcated mode will be added in a future patch-set.
Note that the flexible interface is not used for data path
operations, and thus there are no performance concerns
related to the use of function pointers.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1659f93..8008a50 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 0000000..dd72120
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 0000000..416d1b3
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 61c0182..0e66cdc 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 3f836f3..05fd76c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
-- 
2.6.4

[dpdk-dev] [PATCH v16 08/23] event/dlb: add probe-time hardware init

[Timothy McDaniel]

This commit adds probe-time low level hardware
initialization.  It also adds probe-time init for both
primary and secondary DPDK processes.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 8008a50..57b2837 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 0e66cdc..5502647 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 headers = files()
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..9c4267b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,302 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "../../dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++)
+		hw->domains[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++)
+		hw->rsrcs.ldb_queues[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		hw->rsrcs.ldb_ports[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		hw->rsrcs.dir_pq_pairs[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++)
+		hw->rsrcs.ldb_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++)
+		hw->rsrcs.dir_credit_pools[i].id = i;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+}
+
+void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.vf_to_pf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
index f816acb..87ab8dd 100644
--- a/drivers/event/dlb/pf/dlb_main.c
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 05fd76c..7fc85e9 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -35,9 +35,93 @@
 #include "base/dlb_resource.h"
 
 static void
-dlb_pf_iface_fn_ptrs_init(void)
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
 {
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
 
+	return 0;
+}
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
+
+	return 0;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v16 09/23] event/dlb: add xstats

[Timothy McDaniel]

Add support for DLB xstats.  Perform initialization and add
standard xstats entry points

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1217 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1241 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 57b2837..62b9695 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 0000000..89186d5
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1217 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,		/**< Maximum num of events */
+	inflight_events,		/**< Current num events outstanding */
+	ldb_pool_size,			/**< Num load balanced credits */
+	dir_pool_size,			/**< Num directed credits */
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return __atomic_load_n(&dlb->inflights, __ATOMIC_SEQ_CST);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue = &dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || ids == NULL || xstats_names == NULL)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+				     xs->fn_id);
+			return -EINVAL;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				DLB_LOG_ERR("Unexpected xstat fn_id %d\n",
+					    xs->fn_id);
+				return (uint64_t)-1;
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			DLB_LOG_ERR("Unexpected xstat fn_id %d\n", xs->fn_id);
+			return;
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (f == NULL) {
+		DLB_LOG_ERR("Invalid file pointer\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (dlb == NULL) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id,
+		handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		 fprintf(f, "\tcurrent depth: %u events\n",
+			 dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 5502647..66a3fbe 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
-- 
2.6.4

[dpdk-dev] [PATCH v16 10/23] event/dlb: add infos get and configure

[Timothy McDaniel]

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB
hardware's scheduling domain, such that it is provisioned with
the requested number of ports and queues, provided sufficient
resources are available. Individual queues and ports are
configured later in port setup and eventdev start.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 92341c0..2d7999b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 62b9695..c038794 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -139,6 +139,19 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)
 	return 0;
 }
 
+static void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	rte_free(qm_port->qe4);
+	qm_port->qe4 = NULL;
+
+	rte_free(qm_port->consume_qe);
+	qm_port->consume_qe = NULL;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -231,6 +244,388 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user does not reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
+		dlb->global_dequeue_wait = false;
+	else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WAITPKG)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -309,6 +704,8 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index dd72120..f3e82f2 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 416d1b3..d576232 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 9c4267b..2f8ffec 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -9,107 +9,30 @@
 #include "dlb_osdep_bitmap.h"
 #include "dlb_osdep_types.h"
 #include "dlb_regs.h"
+#include "../../dlb_priv.h"
+#include "../../dlb_inline_fns.h"
 
-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
-{
-	union dlb_dp_dir_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
-
-	r0.field.cfg_vasr_dis = 1;
-
-	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
-}
-
-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
-{
-	union dlb_chp_cfg_chp_csr_ctrl r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
-
-	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
-
-	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
-}
-
-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.ldb_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
 
-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
-{
-	union dlb_sys_cq_mode r0;
-
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
-
-	r0.field.dir_cq64 = 1;
-
-	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
-}
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
 
-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
-{
-	union dlb_sys_sys_alarm_int_enable r0;
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
 
-	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
 
-	r0.field.pf_to_vf_isr_pend_error = 0;
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
 
-	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
-}
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
 
-void dlb_hw_get_num_resources(struct dlb_hw *hw,
-			      struct dlb_get_num_resources_args *arg)
+static inline void dlb_flush_csr(struct dlb_hw *hw)
 {
-	struct dlb_function_resources *rsrcs;
-	struct dlb_bitmap *map;
-
-	rsrcs = &hw->pf;
-
-	arg->num_sched_domains = rsrcs->num_avail_domains;
-
-	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
-
-	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
-
-	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
-
-	map = rsrcs->avail_aqed_freelist_entries;
-
-	arg->num_atomic_inflights = dlb_bitmap_count(map);
-
-	arg->max_contiguous_atomic_inflights =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_hist_list_entries;
-
-	arg->num_hist_list_entries = dlb_bitmap_count(map);
-
-	arg->max_contiguous_hist_list_entries =
-		dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_qed_freelist_entries;
-
-	arg->num_ldb_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
-
-	map = rsrcs->avail_dqed_freelist_entries;
-
-	arg->num_dir_credits = dlb_bitmap_count(map);
-
-	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
-
-	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
-
-	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
 }
 
 static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
@@ -290,6 +213,3997 @@ void dlb_resource_free(struct dlb_hw *hw)
 	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
 }
 
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)
+{
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	return &hw->domains[id];
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (queue == NULL)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id);
+
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (port == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (port == NULL)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (pool == NULL)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)
+{
+	union dlb_sys_sys_alarm_int_enable r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);
+
+	r0.field.pf_to_vf_isr_pend_error = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id, port->id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id, port->id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+							   p->id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+						      p->id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+						       p->id / 4));
+
+	switch (p->id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,
+						  p->id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id,
+					     p->id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,
+					      p->id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id;
+	queue_id = queue->id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = domain->id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = pool->id;
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		dlb_movdir64b(pp_addr, hcw);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id)
+{
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	return &hw->rsrcs.ldb_queues[id];
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid);
+
+		if (queue == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it was not manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));
+
+	return r0.field.count;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		dlb_movdir64b(pp_addr, hcw);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id),
+			   r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	RTE_SET_USED(iter);
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue;
+	RTE_SET_USED(tmp_ldb_queue);
+	struct dlb_dir_pq_pair *tmp_dir_port;
+	RTE_SET_USED(tmp_dir_port);
+	struct dlb_ldb_port *tmp_ldb_port;
+	RTE_SET_USED(tmp_ldb_port);
+	struct dlb_credit_pool *tmp_pool;
+	RTE_SET_USED(tmp_pool);
+	struct dlb_list_entry *iter1;
+	RTE_SET_USED(iter1);
+	struct dlb_list_entry *iter2;
+	RTE_SET_USED(iter2);
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base does not match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain  == NULL || !domain->configured)
+		return -EINVAL;
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process does not decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched
+	 * domain reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void dlb_hw_get_num_resources(struct dlb_hw *hw,
+			      struct dlb_get_num_resources_args *arg)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+}
+
 void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 {
 	union dlb_sys_sys_alarm_int_enable r0;
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 7fc85e9..57a150c 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
 	return 0;
 }
 
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
 static int
 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
 			  uint8_t *revision)
@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 			enum dlb_cq_poll_modes *mode)
 {
@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)
 {
 	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
 	dlb_iface_get_device_version = dlb_pf_get_device_version;
 	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 }
 
-- 
2.6.4

[dpdk-dev] [PATCH v16 11/23] event/dlb: add queue and port default conf

[Timothy McDaniel]

Add support for getting the queue and port default configuration.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c038794..e98a438 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v16 12/23] event/dlb: add queue setup

[Timothy McDaniel]

Load balanced (ldb) queues are setup here.
Directed queues are not set up until link time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index 2d7999b..d8e936a 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e98a438..2b050ad 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -657,6 +657,298 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else
+		sched_type = evq_conf->schedule_type;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = DLB_DEF_UNORDERED_QID_INFLIGHTS;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		DLB_LOG_ERR("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -735,6 +1027,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index f3e82f2..219f79e 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d576232..af1416d 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2f8ffec..35b66e2 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4214,3 +4214,389 @@ void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)
 
 	DLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);
 }
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args)
+{
+	union dlb_sys_ldb_vasqid_v r0 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r1 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r2 = { {0} };
+	union dlb_aqed_pipe_fl_lim r3 = { {0} };
+	union dlb_aqed_pipe_fl_base r4 = { {0} };
+	union dlb_chp_ord_qid_sn_map r7 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r10 = { {0} };
+	union dlb_sys_ldb_qid_v r11 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r5 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r6 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r8 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r9 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+
+	/*
+	 * Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r1.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r1.val);
+
+	r2.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r2.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r2.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id), r2.val);
+
+	r3.field.freelist_disable = 0;
+	r3.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id), r3.val);
+
+	r4.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id), r4.val);
+
+	r5.field.push_ptr = r4.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id), r5.val);
+
+	r6.field.pop_ptr = r4.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id), r6.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r7.field.mode = sn_group->mode;
+	r7.field.slot = queue->sn_slot;
+	r7.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id), r7.val);
+
+	/*
+	 * This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r8.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id), r8.val);
+
+	r9.field.group = sn_group->id;
+	r9.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id), r9.val);
+
+	r10.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r10.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id), r10.val);
+
+	r11.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id), r11.val);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = queue->id;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 57a150c..fffb88b 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -198,6 +198,83 @@ dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 	return 0;
 }
 
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -209,7 +286,11 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v16 13/23] event/dlb: add port setup

[Timothy McDaniel]

Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d8e936a..bb3455b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2b050ad..09e4640 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -152,6 +152,69 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 	qm_port->consume_qe = NULL;
 }
 
+static int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_zmalloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -657,6 +720,329 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "ldb_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be at least %d\n",
+			    DLB_MIN_DIR_CQ_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "dir_port%d",
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -909,7 +1295,7 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	struct dlb_eventdev_queue *ev_queue;
 	int ret;
 
-	if (!queue_conf)
+	if (queue_conf == NULL)
 		return -EINVAL;
 
 	if (ev_qid >= dlb->num_queues)
@@ -949,6 +1335,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1028,6 +1541,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.port_setup       = dlb_eventdev_port_setup,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 219f79e..fbbf9d7 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index af1416d..d578185 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 35b66e2..799cb2b 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -4455,7 +4455,7 @@ dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
 
-	if (!domain) {
+	if (domain == NULL) {
 		resp->status = DLB_ST_INVALID_DOMAIN_ID;
 		return -1;
 	}
@@ -4557,7 +4557,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 		return -EINVAL;
 
 	domain = dlb_get_domain_from_id(hw, domain_id);
-	if (!domain) {
+	if (domain == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: domain not found\n",
 			   __func__, __LINE__);
@@ -4567,7 +4567,7 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
 
 	/* Verification should catch this. */
-	if (!queue) {
+	if (queue == NULL) {
 		DLB_HW_ERR(hw,
 			   "[%s():%d] Internal error: no available ldb queues\n",
 			   __func__, __LINE__);
@@ -4600,3 +4600,1433 @@ int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
 
 	return 0;
 }
+
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id, domain);
+
+		if (port  == NULL || port->domain_id != domain->id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	union dlb_sys_dir_qid_v r1 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id * DLB_MAX_NUM_DIR_PORTS) + queue->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	r1.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id), r1.val);
+
+	queue->queue_configured = true;
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id, domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = queue->id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_credit_pool *pool;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if (pool->id == id)
+			return pool;
+
+	return NULL;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id), r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id), r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id), r18.val);
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   domain);
+
+		if (queue  == NULL || queue->domain_id != domain->id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       domain);
+
+		if (pool  == NULL || !pool->configured ||
+		    pool->domain_id != domain->id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       domain);
+
+	if (pool  == NULL || !pool->configured ||
+	    pool->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   domain);
+		if (dir_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id),
+		   r1.val);
+
+	r2.field.is_pf = 1;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id),
+		   r2.val);
+
+	r3.field.vas = domain->id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id),
+		   r3.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),
+		   r18.val);
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id), r1.val);
+
+	r2.field.is_pf = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   domain);
+		if (ldb_pool == NULL) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id, domain);
+	if (dir_pool == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw, domain, port, args);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = port->id;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fffb88b..5e14271 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -221,6 +221,213 @@ dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(const struct rte_memzone **mz, rte_iova_t *phys,
+			   size_t size, int align)
+{
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb_port_mem_%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_main_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	*mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (*mz == NULL) {
+		DLB_LOG_ERR("Unable to allocate DMA memory of size %zu bytes\n",
+			    size);
+		*phys = 0;
+		return NULL;
+	}
+	*phys = (*mz)->iova;
+	return (*mz)->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz, cq_alloc_depth;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* The hardware always uses a CQ depth of at least
+	 * DLB_MIN_HARDWARE_CQ_DEPTH, even though from the user
+	 * perspective we support a depth as low as 1 for LDB ports.
+	 */
+	cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB_MIN_HARDWARE_CQ_DEPTH);
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cq_alloc_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_LDB].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+	uint8_t *port_base;
+	const struct rte_memzone *mz;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&mz, &pc_dma_base,
+					       alloc_sz, PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb pf pmd could not lock page for device i/o\n");
+		goto create_port_err;
+	}
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+	dlb_port[response.id][DLB_DIR].mz = mz;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	rte_memzone_free(mz);
+
+	return ret;
+}
+
+static int
 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 			 struct dlb_get_sn_allocation_args *args)
 {
@@ -287,6 +494,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v16 14/23] event/dlb: add port link

[Timothy McDaniel]

Add port link entry point. Directed queues are identified and created
at this stage. Their setup deferred until link-time, at which
point we know the directed port ID. Directed queue setup
will only fail if this queue is already setup or there are
no directed queues left to configure.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 09e4640..29d5a0c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index fbbf9d7..aaf4506 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index d578185..c0f5f2e 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 799cb2b..2d0b1d0 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6030,3 +6030,644 @@ int dlb_hw_create_dir_port(struct dlb_hw *hw,
 	return 0;
 }
 
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id, struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_port *port;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if (port->id == id)
+			return port;
+
+	return NULL;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, domain);
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_ldb_queue *queue;
+	RTE_SET_USED(iter);
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if (queue->id == id)
+			return queue;
+
+	return NULL;
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue == NULL || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available CQ slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asynchronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n", args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n", args->priority);
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+
+	if (port  == NULL || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+
+	if (queue  == NULL || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id != domain->id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw, domain_id, args, resp))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, domain);
+	if (port == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, domain);
+	if (queue == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 5e14271..fed6719 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -482,6 +482,72 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -497,6 +563,9 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
 	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v16 15/23] event/dlb: add port unlink and port unlinks in progress

[Timothy McDaniel]

Add supports for the port unlink(s) eventdev entry points.
The unlink operation is an asynchronous operation executed by
a control thread, and the unlinks-in-progress function reads
a counter shared with the control thread. Port QE and memzone
memory is freed here.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 29d5a0c..748789c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

[dpdk-dev] [PATCH v16 16/23] event/dlb: add eventdev start

[Timothy McDaniel]

Add support for the eventdev start entry point.
DLB delays setting up single link resources until
eventdev start, because it is only then that it can
ascertain which ports have just one linked queue.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 748789c..ea57c28 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index aaf4506..22d524b 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index c0f5f2e..8c905ab 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 2d0b1d0..6dad99d 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6410,6 +6410,32 @@ static int dlb_verify_map_qid_args(struct dlb_hw *hw,
 	return 0;
 }
 
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
 static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
 					     struct dlb_ldb_queue *queue,
 					     struct dlb_cmd_response *resp)
@@ -6671,3 +6697,119 @@ int dlb_hw_map_qid(struct dlb_hw *hw,
 	return 0;
 }
 
+static void dlb_log_start_domain(struct dlb_hw *hw, u32 domain_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),
+		   r0.val);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter;
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	RTE_SET_USED(arg);
+	RTE_SET_USED(iter);
+
+	dlb_log_start_domain(hw, domain_id);
+
+	if (dlb_verify_start_domain_args(hw, domain_id, resp))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_LDB_QUEUES + ldb_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id * DLB_MAX_NUM_DIR_PORTS + dir_queue->id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index fed6719..1d2e133 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
-- 
2.6.4

[dpdk-dev] [PATCH v16 17/23] event/dlb: add enqueue and its burst variants

[Timothy McDaniel]

Add support for enqueue and its variants.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 161 ++++++++++++
 drivers/event/dlb/dlb.c      | 567 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 728 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index bb3455b..ae126c4 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index ea57c28..4053679 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2135,6 +2135,568 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = __atomic_load_n(&dlb->inflights,
+						__ATOMIC_SEQ_CST);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+		    dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		__atomic_fetch_add(&dlb->inflights, credit_update_quanta,
+				   __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		/* Replenish credits, saving one quanta for enqueues */
+		uint16_t val = ev_port->inflight_credits - quanta;
+
+		__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);
+		ev_port->inflight_credits -= val;
+	}
+}
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	case 0:
+		break;
+	}
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	dlb_movdir64b(port_data->pp_addr, qe4);
+}
+
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  bool do_sfence,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	if (do_sfence)
+		rte_wmb();
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	i = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	while (i < num) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int pop_offs = 0;
+		int j = 0;
+
+		memset(qm_port->qe4,
+		       0,
+		       DLB_NUM_QES_PER_CACHE_LINE *
+		       sizeof(struct dlb_enqueue_qe));
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j - pop_offs,
+				     sched_types, queue_ids);
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the enqueue count */
+		i += j - pop_offs;
+
+		/* Don't interpret j < DLB_NUM_... as out-of-credits if
+		 * pop_offs != 0
+		 */
+		if (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
+			break;
+	}
+
+	RTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));
+
+	return i;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			const struct rte_event events[],
+			uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			    const struct rte_event events[],
+			    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -2159,6 +2721,11 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 
 	/* Expose PMD's eventdev interface */
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 }
 
 int
-- 
2.6.4

[dpdk-dev] [PATCH v16 18/23] event/dlb: add dequeue and its burst variants

[Timothy McDaniel]

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses
umonitor/umwait if supported by the processor. This allows
the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the
hardware will pack 4 QEs into each cache line. In sparse cq
mode, the hardware will only populate one QE per cache line.
Software must be aware of the cq mode, and take the appropriate
actions, based on the mode.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 719 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 740 insertions(+)

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index ae126c4..4c4f56b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 4053679..1aab7ce 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -25,6 +25,7 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_mbuf.h>
+#include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
@@ -2605,6 +2606,46 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,
 	dlb_pp_write(qm_port->qe4, port_data);
 }
 
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(port_data->pp_addr, qe);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
@@ -2697,9 +2738,678 @@ dlb_event_enqueue_forward_burst(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	rte_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+		num++;
+	}
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num);
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define DLB_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     DLB_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define DLB_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     DLB_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     DLB_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define DLB_EVENT_EV_TYPE_DW 0
+#define DLB_EVENT_EV_TYPE_SHIFT 28
+#define DLB_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << DLB_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << DLB_EVENT_SUB_EV_TYPE_SHIFT,
+			DLB_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define DLB_EVENT_SCHED_TYPE_BYTE 4
+#define DLB_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << DLB_EVENT_SCHED_TYPE_SHIFT,
+		DLB_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+		union {
+			uint64_t raw_qe[2];
+			struct dlb_dequeue_qe qe;
+		} qe_mask;
+		uint64_t expected_value;
+		volatile uint64_t *monitor_addr;
+
+		qe_mask.qe.cq_gen = 1; /* set mask */
+
+		cq_base = port_data->cq_base;
+		monitor_addr = (volatile uint64_t *)(volatile void *)
+			&cq_base[qm_port->cq_idx];
+		monitor_addr++; /* cq_gen bit is in second 64bit location */
+
+		if (qm_port->gen_bit)
+			expected_value = qe_mask.raw_qe[1];
+		else
+			expected_value = 0;
+
+		rte_power_monitor(monitor_addr, expected_value,
+				  qe_mask.raw_qe[1], timeout + start_ticks,
+				  sizeof(uint64_t));
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static int
+dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return rte_errno;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	i = 0;
+
+	if (qm_port->is_directed) {
+		i = n;
+		goto sw_credit_update;
+	}
+
+	while (i < n) {
+		int pop_offs = 0;
+		int j = 0;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+			qm_port->issued_releases++;
+		}
+
+		dlb_hw_do_enqueue(qm_port, i == 0, port_data);
+
+		/* Don't include the token pop QE in the release count */
+		i += j - pop_offs;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases) {
+		DLB_LOG_ERR("Unrecoverable application error. Outstanding releases underflowed.\n");
+		rte_errno = -ENOTRECOVERABLE;
+		return rte_errno;
+	}
+
+	ev_port->outstanding_releases -= i;
+	ev_port->inflight_credits += i;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+	return 0;
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	uint16_t cnt;
+	int ret;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		ret = dlb_event_release(dlb, ev_port->id, out_rels);
+		if (ret)
+			return(ret);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2726,6 +3436,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
+
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 int
-- 
2.6.4

Re: [dpdk-dev] [PATCH v16 18/23] event/dlb: add dequeue and its burst variants

[Burakov, Anatoly]

On 01-Nov-20 11:30 PM, Timothy McDaniel wrote:
> Add support for dequeue, dequeue_burst, ...
> 
> DLB does not currently support interrupts, but instead uses
> umonitor/umwait if supported by the processor. This allows
> the software to monitor and wait on writes to a cache-line.
> 
> DLB supports normal and sparse cq mode. In normal mode the
> hardware will pack 4 QEs into each cache line. In sparse cq
> mode, the hardware will only populate one QE per cache line.
> Software must be aware of the cq mode, and take the appropriate
> actions, based on the mode.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> Reviewed-by: Gage Eads <gage.eads@intel.com>
> ---

<snip>

> +static inline int
> +dlb_dequeue_wait(struct dlb_eventdev *dlb,
> +		 struct dlb_eventdev_port *ev_port,
> +		 struct dlb_port *qm_port,
> +		 uint64_t timeout,
> +		 uint64_t start_ticks)
> +{
> +	struct process_local_port_data *port_data;
> +	uint64_t elapsed_ticks;
> +
> +	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
> +
> +	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
> +
> +	/* Wait/poll time expired */
> +	if (elapsed_ticks >= timeout) {
> +		/* Interrupts not supported by PF PMD */
> +		return 1;
> +	} else if (dlb->umwait_allowed) {
> +		volatile struct dlb_dequeue_qe *cq_base;
> +		union {
> +			uint64_t raw_qe[2];
> +			struct dlb_dequeue_qe qe;
> +		} qe_mask;
> +		uint64_t expected_value;
> +		volatile uint64_t *monitor_addr;
> +
> +		qe_mask.qe.cq_gen = 1; /* set mask */
> +
> +		cq_base = port_data->cq_base;
> +		monitor_addr = (volatile uint64_t *)(volatile void *)
> +			&cq_base[qm_port->cq_idx];
> +		monitor_addr++; /* cq_gen bit is in second 64bit location */
> +
> +		if (qm_port->gen_bit)
> +			expected_value = qe_mask.raw_qe[1];
> +		else
> +			expected_value = 0;
> +
> +		rte_power_monitor(monitor_addr, expected_value,
> +				  qe_mask.raw_qe[1], timeout + start_ticks,
> +				  sizeof(uint64_t));
> +
> +		DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait, 1);
> +	} else {
> +		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
> +		uint64_t curr_ticks = rte_get_timer_cycles();
> +		uint64_t init_ticks = curr_ticks;
> +
> +		while ((curr_ticks - start_ticks < timeout) &&
> +		       (curr_ticks - init_ticks < poll_interval))
> +			curr_ticks = rte_get_timer_cycles();
> +	}
> +
> +	return 0;

For UMONITOR code, LGTM.

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

[dpdk-dev] [PATCH v16 19/23] event/dlb: add eventdev stop and close

[Timothy McDaniel]

Add support for eventdev stop and close entry points.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 1aab7ce..95b7bbe 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -72,17 +72,6 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3405,6 +3394,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3414,6 +3646,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
index 22d524b..44f958f 100644
--- a/drivers/event/dlb/dlb_iface.c
+++ b/drivers/event/dlb/dlb_iface.c
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
index 8c905ab..9f61135 100644
--- a/drivers/event/dlb/dlb_iface.h
+++ b/drivers/event/dlb/dlb_iface.h
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
index 6dad99d..4984de5 100644
--- a/drivers/event/dlb/pf/base/dlb_resource.c
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -6813,3 +6813,92 @@ int dlb_hw_start_domain(struct dlb_hw *hw,
 
 	return 0;
 }
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id);
+	if (domain == NULL) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, domain);
+	if (queue == NULL) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
index 1d2e133..cf88c49 100644
--- a/drivers/event/dlb/pf/dlb_pf.c
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -570,6 +570,50 @@ dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
 static void
 dlb_pf_iface_fn_ptrs_init(void)
 {
@@ -589,10 +633,13 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
 	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+
 }
 
 /* PCI DEV HOOKS */
-- 
2.6.4

[dpdk-dev] [PATCH v16 20/23] event/dlb: add PMD's token pop public interface

[Timothy McDaniel]

The PMD uses a public interface to allow applications to
control the token pop mode. Supported token pop modes are
as follows, and they impact core scheduling affinity for
ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
	     are released. Supported on load-balanced ports
	     only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 doc/api/doxy-api-index.md       |   3 +-
 doc/api/doxy-api.conf.in        |   1 +
 drivers/event/dlb/dlb.c         | 187 ++++++++++++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   5 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 ++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 8 files changed, 309 insertions(+), 11 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index b865a51..de2d113 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -52,7 +52,8 @@ The public API headers are grouped by topics:
   [dpaa2_mempool]      (@ref rte_dpaa2_mempool.h),
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
-  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h),
+  [dlb]                (@ref rte_pmd_dlb.h)
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index c5b01a1..116acf6 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -7,6 +7,7 @@ USE_MDFILE_AS_MAINPAGE  = @TOPDIR@/doc/api/doxy-api-index.md
 INPUT                   = @TOPDIR@/doc/api/doxy-api-index.md \
                           @TOPDIR@/drivers/bus/vdev \
                           @TOPDIR@/drivers/crypto/scheduler \
+                          @TOPDIR@/drivers/event/dlb \
                           @TOPDIR@/drivers/mempool/dpaa2 \
                           @TOPDIR@/drivers/net/ark \
                           @TOPDIR@/drivers/net/bnxt \
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 95b7bbe..e59a0de 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -72,6 +72,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num);
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -1003,6 +1022,33 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -1163,6 +1209,8 @@ dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
 
 	qm_port->dequeue_depth = dequeue_depth;
 
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
 	qm_port->owed_tokens = 0;
 	qm_port->issued_releases = 0;
 
@@ -2572,6 +2620,30 @@ dlb_event_build_hcws(struct dlb_port *qm_port,
 	}
 }
 
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+	qm_port->owed_tokens = 0;
+}
+
 static __rte_always_inline void
 dlb_pp_write(struct dlb_enqueue_qe *qe4,
 	     struct process_local_port_data *port_data)
@@ -2638,7 +2710,8 @@ dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
 static inline uint16_t
 __dlb_event_enqueue_burst(void *event_port,
 			  const struct rte_event events[],
-			  uint16_t num)
+			  uint16_t num,
+			  bool use_delayed)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
 	struct dlb_port *qm_port = &ev_port->qm_port;
@@ -2666,6 +2739,35 @@ __dlb_event_enqueue_burst(void *event_port,
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
 			const struct rte_event *ev = &events[i + j];
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (use_delayed &&
+			    qm_port->token_pop_mode == DELAYED_POP &&
+			    (ev->op == RTE_EVENT_OP_FORWARD ||
+			     ev->op == RTE_EVENT_OP_RELEASE) &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE and break out. This
+				 * may result in a partial HCW, but that is
+				 * simpler than supporting arbitrary QE
+				 * insertion.
+				 */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
 						   port_data, &sched_types[j],
@@ -2701,14 +2803,29 @@ dlb_event_enqueue_burst(void *event_port,
 			const struct rte_event events[],
 			uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				const struct rte_event events[],
+				uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static inline uint16_t
 dlb_event_enqueue(void *event_port,
 		  const struct rte_event events[])
 {
-	return __dlb_event_enqueue_burst(event_port, events, 1);
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			  const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
 }
 
 static uint16_t
@@ -2716,7 +2833,15 @@ dlb_event_enqueue_new_burst(void *event_port,
 			    const struct rte_event events[],
 			    uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				    const struct rte_event events[],
+				    uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static uint16_t
@@ -2724,7 +2849,15 @@ dlb_event_enqueue_forward_burst(void *event_port,
 				const struct rte_event events[],
 				uint16_t num)
 {
-	return __dlb_event_enqueue_burst(event_port, events, num);
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					const struct rte_event events[],
+					uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
 static __rte_always_inline int
@@ -3124,7 +3257,8 @@ dlb_hw_dequeue(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3249,7 +3383,8 @@ dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
 
 	qm_port->owed_tokens += num;
 
-	dlb_consume_qe_immediate(qm_port, num);
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
 
 	ev_port->outstanding_releases += num;
 
@@ -3293,6 +3428,28 @@ dlb_event_release(struct dlb_eventdev *dlb, uint8_t port_id, int n)
 		qm_port->qe4[3].cmd_byte = 0;
 
 		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
+			int16_t thresh = qm_port->token_pop_thresh;
+
+			if (qm_port->token_pop_mode == DELAYED_POP &&
+			    qm_port->issued_releases >= thresh - 1) {
+				/* Insert the token pop QE */
+				dlb_construct_token_pop_qe(qm_port, j);
+
+				/* Reset the releases for the next QE batch */
+				qm_port->issued_releases -= thresh;
+
+				/* When using delayed token pop mode, the
+				 * initial token threshold is the full CQ
+				 * depth. After the first token pop, we need to
+				 * reset it to the dequeue_depth.
+				 */
+				qm_port->token_pop_thresh =
+					qm_port->dequeue_depth;
+
+				pop_offs = 1;
+				j++;
+				break;
+			}
 
 			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
 			qm_port->issued_releases++;
@@ -3325,6 +3482,7 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 			uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3344,6 +3502,10 @@ dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3362,6 +3524,7 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 			       uint16_t num, uint64_t wait)
 {
 	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
 	struct dlb_eventdev *dlb = ev_port->dlb;
 	uint16_t cnt;
 	int ret;
@@ -3381,6 +3544,10 @@ dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
 		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
 	}
 
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
 	cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
 
 	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
@@ -3687,7 +3854,7 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   struct dlb_devargs *dlb_args)
 {
 	struct dlb_eventdev *dlb;
-	int err;
+	int err, i;
 
 	dlb = dev->data->dev_private;
 
@@ -3736,6 +3903,10 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
index adb1f7a..58ff428 100644
--- a/drivers/event/dlb/dlb_priv.h
+++ b/drivers/event/dlb/dlb_priv.h
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 66a3fbe..2ea3b4c 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -12,9 +12,10 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
-headers = files()
+headers = files('rte_pmd_dlb.h')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 0000000..bc802d3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 0000000..9cf6dd3
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ *
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+__rte_experimental
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/version.map b/drivers/event/dlb/version.map
index 4a76d1d..3338a22 100644
--- a/drivers/event/dlb/version.map
+++ b/drivers/event/dlb/version.map
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
-- 
2.6.4

[dpdk-dev] [PATCH v16 21/23] event/dlb: add PMD self-tests

[Timothy McDaniel]

Add a variety of self-tests for both ldb and directed
ports/queues, as well as configure, start, stop, link, etc...

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/app/test/test_eventdev.c b/app/test/test_eventdev.c
index 62019c1..ba27bed 100644
--- a/app/test/test_eventdev.c
+++ b/app/test/test_eventdev.c
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index e59a0de..afef56c 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3828,6 +3828,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 0000000..b9ef778
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1539 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+/* destruction */
+static inline int
+cleanup(void)
+{
+	rte_event_dev_stop(evdev);
+	return rte_event_dev_close(evdev);
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	return rte_event_dev_close(evdev);
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_directed_traffic(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_deferred_sched(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+test_delayed_pop(void)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup();
+	return 0;
+
+err:
+	cleanup();
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link();
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic();
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic();
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched();
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop();
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "dlb_event";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 2ea3b4c..7d40c16 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 headers = files('rte_pmd_dlb.h')
-- 
2.6.4

[dpdk-dev] [PATCH v16 22/23] event/dlb: add queue and port release

[Timothy McDaniel]

These entry points are NO-OPS. DLB does not support
reconfiguring individual queues or ports. The entire device
must be reconfigured.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index afef56c..bda696a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3804,6 +3807,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3818,7 +3843,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
-- 
2.6.4

[dpdk-dev] [PATCH v16 23/23] event/dlb: add timeout ticks entry point

[Timothy McDaniel]

Adds the timeout ticks conversion function.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index bda696a..2bb270d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -3829,6 +3829,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3850,6 +3862,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
-- 
2.6.4

Re: [dpdk-dev] [PATCH v16 00/23] Add DLB PMD

[Jerin Jacob]

On Mon, Nov 2, 2020 at 4:58 AM Timothy McDaniel
<timothy.mcdaniel@intel.com> wrote:
>
> The following patch series adds support for a new eventdev PMD. The DLB
> PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware.
> The DLB is a PCIe device that provides load-balanced, prioritized
> scheduling of core-to-core communication. The device consists of
> queues and arbiters that connect producer and consumer cores, and
> implements load-balanced queueing features including:
> - Lock-free multi-producer/multi-consumer operation.
> - Multiple priority levels for varying traffic types.
> - 'Direct' traffic (i.e. multi-producer/single-consumer)
> - Simple unordered load-balanced distribution.
> - Atomic lock-free load balancing across multiple consumers.
> - Queue element reordering feature allowing ordered load-balanced
>   distribution.
>
> The DLB hardware supports both load balanced and directed ports and
> queues. Unlike other eventdev devices already in the repo,  not all
> DLB ports and queues are equally capable. In particular, directed
> ports are limited to a single link, and must be connected to a directed
> queue.
> Additionally, even though LDB ports may link multiple queues, the
> number of queues that may be linked is limited by hardware. Another
> difference is that DLB does not have a straightforward way of carrying
> the flow_id in the queue elements (QE) that the hardware operates on.
>
> While reviewing the code, please be aware that this PMD has full
> control over the DLB hardware. Intel will be extending the DLB PMD
> in the future (not as part of this first series) with a mode that we
> refer to as the bifurcated PMD. The bifurcated PMD communicates with a
> kernel driver to configure the device, ports, and queues, and memory
> maps device MMIO so datapath operations occur purely in user-space.
>
> The framework to support both the PF PMD and bifurcated PMD exists in
> this patchset, and is why the iface.[ch] layer is present.


Series applied to dpdk-next-eventdev/for-main with the following fix. Thanks.

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
index d44afcdcf..4c4f56b2b 100644
--- a/doc/guides/eventdevs/dlb.rst
+++ b/doc/guides/eventdevs/dlb.rst
@@ -2,7 +2,7 @@
     Copyright(c) 2020 Intel Corporation.

 Driver for the Intel® Dynamic Load Balancer (DLB)
+=================================================
-==================================================

 The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.



>
> Major changes in V16
> ====================
> Address additional comments from David Marchand:
> - converted printfs in dlb/pf/dlb_main.c to DLB_LOG
> - fixed a repeated word error in dlb/pf/base/osdep_bitmap.h
> - caught up with marking the patches that Gage reviewed
>
> Major changes in V15
> ====================
> Address comments from David Marchand:
> - this patch-set is based on Nov 1, 2020 dpdk-next-eventdev
> - fix docs build (doxy-api.conf.in and doxy-api-index.md)
> - restore blank line in MAINTAINERS file
> - move dlb announcement in release_20_11.rst after ethdev
> - use headers = files() for exported meson public headers
> - fix a typo in 'add documentation ..." commit message
> - use eal version of cldemote
> - convert a couple of printfs to LOG messages
> - fix missing "~" in dlb documentation
> - delay introduction of _delayed token pop functions to
>   token pop commit (fixes 8 or so unused function errors)
> - all patches build incrementally (gcc), and checkpatches reports
>   success
> - I am not able to run clang locally. If clang errors are still
>   present I will ask IT to install clang on a build server tomorrow.
>
> Major changes in V14
> ====================
> - Fixed format errors in doc/api/doxy-api-index.md
> - Delayed introduction of dlb2_consume_qe_immediate until
>   add-dequeue-and-its-burst-variants.patch
> - Delayed introduction of dlb2_construct_token_pop_qe until
>   add-PMD-s-token-pop-public-interface.patch
> - Delayed introduction of dlb_equeue_*_delayed until
>   add dequeue and its burst variants.patch
>
> Major changes in V13
> ====================
> - removed now unused functions dlb_umwait and dlb_umonitor
>
> Major changes in V12
> ====================
> - Fix CENTOS build error: use __m128i instead of __v2di with
>   _mm_stream_si128
>
> Major changes in V11
> ====================
> - removed unused function, fixing build error
> - fixed typo in port_setup commit message
> - this patch series is based on dpdk-next-eventdev
>
> Major changes in v10
> =====================
> - convert to use rte_power_monitor patches
> - replace __builtin_ia32_movntdq() with _mm_stream_si128()
> - remove unused functions in dlb_selftest.c
>
> Major changes in v9
> =====================
> - fixed a build error due to __rte_cache_aligned being placed after
>   the ";" character, instead of before it.
>
> Major changes in v8 after dpdk reviews
> =====================
> - moved introduction of dlb in relnotes_20_11 to first patch in series
> - fixed underlines in dlb.rst that were too short
> - note that the code still uses its private byte-encoded versions of
>   umonitor/umwait, rather than the new functions in the power
>   patch that are built on top of those intrinsics. This is intentional.
>
> Major changes in v7 after dpdk reviews
> =====================
> - updated MAINTAINERS file to alphabetically insert DLB
> - don't create RTE_ symbols in PMD
> - converted to use version.map scheme
> - converted to use .._master_lcore instead of .._main_lcore
> - this patch set is based on dpdk-next-eventdev
>
> Major changes in v6 after dpdk reviews:
> =====================
> - fixed meson conditional build. Moved test into driver’s meson.build
>   file instead of event/meson.build
> - documentation is populated as associated code is introduced
> - add log_register in add dynamic logging patch
> - rename RTE_xxx symbol(s) as DLB2_xxx
> - replaced function ptr enqueue_four with direct call to movdir64b
> - remove unused port_pages
> - broke up probe patch into 3 smaller patches for easier review
> - changed param order of movdir64b/movntdq to match intrinsics
> - added self to MAINTAINERS files
> - squashed announcement of availability into last patch in series
> - correct spelling errors and delete repeated words
> - DPDK_21.0 -> DPDK 21 in map file
> - add experimental banner to public structs and APIs
> - implemented other suggestions from code reviews of DLB2 PMD. The
>   software is very similar in form so some DLB2 reviews comments
>   were applicable to DLB as well
>
> Major changes in v5 after dpdk reviews and additional internal reviews
> by colleagues at Intel:
> ================
> - implement changes requested in code reviews by Gage Eads and Mike Chen
> - fix a memzone leak
> - convert to use eal rte-cpuflags patch from Liang Ma
>
> Major changes in v4 after dpdk reviews and additional internal reviews
> by colleagues at Intel:
> ================
> - Remove make infrastructure
> - shared code (pf/base) is now added incrementally
> - flexible interface (iface.[ch]) is now added incrementally
> - removed calls to rte_panic
> - do not call pthread_create directly
> - remove unused internal API, os_time
> - convert rte_atomic to __atomic builtins
> - broke out eventdev ABI changes, test/api changes, and new internal PCI
>   named probe API
> - relocated enqueue logic to enqueue patch
>
> Major Changes in V3:
> ================
> - Fixed a memory corruption issue due to not allocating enough CQ
> memory for depths < 8. Hardware requires minimum allocation to be
> at least 8 entries.
> - Address review comments from Gage and Mattias.
> - Remove versioning
> - minor formatting changes
>
> Major changes in V2:
> ================
> - Correct ABI break that was present in V1.
> - Address some of the review comments received from Mattias.
>   I will address the remaining items identified by Mattias in the next
>   patch delivery.
> - General code cleanup based on internal code reviews
>
> Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")
>
> Timothy McDaniel (23):
>   event/dlb: add documentation and meson infrastructure
>   event/dlb: add dynamic logging
>   event/dlb: add private data structures and constants
>   event/dlb: add definitions shared with LKM or shared code
>   event/dlb: add inline functions
>   event/dlb: add eventdev probe
>   event/dlb: add flexible interface
>   event/dlb: add probe-time hardware init
>   event/dlb: add xstats
>   event/dlb: add infos get and configure
>   event/dlb: add queue and port default conf
>   event/dlb: add queue setup
>   event/dlb: add port setup
>   event/dlb: add port link
>   event/dlb: add port unlink and port unlinks in progress
>   event/dlb: add eventdev start
>   event/dlb: add enqueue and its burst variants
>   event/dlb: add dequeue and its burst variants
>   event/dlb: add eventdev stop and close
>   event/dlb: add PMD's token pop public interface
>   event/dlb: add PMD self-tests
>   event/dlb: add queue and port release
>   event/dlb: add timeout ticks entry point
>
>  MAINTAINERS                                  |    5 +
>  app/test/test_eventdev.c                     |    7 +
>  config/rte_config.h                          |    6 +
>  doc/api/doxy-api-index.md                    |    3 +-
>  doc/api/doxy-api.conf.in                     |    1 +
>  doc/guides/eventdevs/dlb.rst                 |  341 ++
>  doc/guides/eventdevs/index.rst               |    1 +
>  doc/guides/rel_notes/release_20_11.rst       |    5 +
>  drivers/event/dlb/dlb.c                      | 4079 +++++++++++++++
>  drivers/event/dlb/dlb_iface.c                |   79 +
>  drivers/event/dlb/dlb_iface.h                |   82 +
>  drivers/event/dlb/dlb_inline_fns.h           |   36 +
>  drivers/event/dlb/dlb_log.h                  |   25 +
>  drivers/event/dlb/dlb_priv.h                 |  513 ++
>  drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
>  drivers/event/dlb/dlb_user.h                 |  814 +++
>  drivers/event/dlb/dlb_xstats.c               | 1217 +++++
>  drivers/event/dlb/meson.build                |   22 +
>  drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
>  drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
>  drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
>  drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
>  drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
>  drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
>  drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
>  drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
>  drivers/event/dlb/pf/dlb_main.c              |  586 +++
>  drivers/event/dlb/pf/dlb_main.h              |   47 +
>  drivers/event/dlb/pf/dlb_pf.c                |  750 +++
>  drivers/event/dlb/rte_pmd_dlb.c              |   38 +
>  drivers/event/dlb/rte_pmd_dlb.h              |   77 +
>  drivers/event/dlb/version.map                |    9 +
>  drivers/event/meson.build                    |    2 +-
>  33 files changed, 21677 insertions(+), 2 deletions(-)
>  create mode 100644 doc/guides/eventdevs/dlb.rst
>  create mode 100644 drivers/event/dlb/dlb.c
>  create mode 100644 drivers/event/dlb/dlb_iface.c
>  create mode 100644 drivers/event/dlb/dlb_iface.h
>  create mode 100644 drivers/event/dlb/dlb_inline_fns.h
>  create mode 100644 drivers/event/dlb/dlb_log.h
>  create mode 100644 drivers/event/dlb/dlb_priv.h
>  create mode 100644 drivers/event/dlb/dlb_selftest.c
>  create mode 100644 drivers/event/dlb/dlb_user.h
>  create mode 100644 drivers/event/dlb/dlb_xstats.c
>  create mode 100644 drivers/event/dlb/meson.build
>  create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
>  create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
>  create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
>  create mode 100644 drivers/event/dlb/pf/dlb_main.c
>  create mode 100644 drivers/event/dlb/pf/dlb_main.h
>  create mode 100644 drivers/event/dlb/pf/dlb_pf.c
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
>  create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
>  create mode 100644 drivers/event/dlb/version.map
>
> --
> 2.6.4
>

[dpdk-dev] [PATCH 02/27] eventdev: do not pass disable_implicit_release bit to trace macro

[McDaniel, Timothy]

    The disable implicit_release indicator is now a bitfield, so it
    must be converted to uint8_t when calling the tracepoint macro.

Change-Id: Iab83bbc295de2431a936ce2a73ed6801366d392f
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 lib/librte_eventdev/rte_event_crypto_adapter.c |  7 ++++++-
 lib/librte_eventdev/rte_event_eth_tx_adapter.c | 14 ++++++++++++--
 lib/librte_eventdev/rte_eventdev.c             | 10 +++++++---
 lib/librte_eventdev/rte_eventdev_trace.h       | 15 +++++++++------
 4 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/lib/librte_eventdev/rte_event_crypto_adapter.c b/lib/librte_eventdev/rte_event_crypto_adapter.c
index 8f2593316..1b56f2eb4 100644
--- a/lib/librte_eventdev/rte_event_crypto_adapter.c
+++ b/lib/librte_eventdev/rte_event_crypto_adapter.c
@@ -205,8 +205,10 @@ rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
 	struct rte_event_crypto_adapter *adapter;
 	char mem_name[CRYPTO_ADAPTER_NAME_LEN];
 	struct rte_event_dev_info dev_info;
+	struct rte_event_port_conf *port_conf = conf_arg;
 	int socket_id;
 	uint8_t i;
+	uint8_t disable_impl_rel;
 	int ret;
 
 	EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
@@ -268,8 +270,11 @@ rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
 
 	event_crypto_adapter[id] = adapter;
 
+	disable_impl_rel = !!(port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+
 	rte_eventdev_trace_crypto_adapter_create(id, dev_id, adapter, conf_arg,
-		mode);
+		disable_impl_rel, mode);
 	return 0;
 }
 
diff --git a/lib/librte_eventdev/rte_event_eth_tx_adapter.c b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
index 8a72256de..e872f7c8c 100644
--- a/lib/librte_eventdev/rte_event_eth_tx_adapter.c
+++ b/lib/librte_eventdev/rte_event_eth_tx_adapter.c
@@ -910,6 +910,7 @@ rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
 				struct rte_event_port_conf *port_conf)
 {
 	struct rte_eventdev *dev;
+	uint8_t disable_impl_rel;
 	int ret;
 
 	if (port_conf == NULL)
@@ -943,8 +944,12 @@ rte_event_eth_tx_adapter_create(uint8_t id, uint8_t dev_id,
 		txa_dev_id_array[id] = TXA_INVALID_DEV_ID;
 		return ret;
 	}
+
+	disable_impl_rel = !!(port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+
 	rte_eventdev_trace_eth_tx_adapter_create(id, dev_id, NULL, port_conf,
-		ret);
+		disable_impl_rel, ret);
 	txa_dev_id_array[id] = dev_id;
 	return 0;
 }
@@ -955,6 +960,8 @@ rte_event_eth_tx_adapter_create_ext(uint8_t id, uint8_t dev_id,
 				void *conf_arg)
 {
 	struct rte_eventdev *dev;
+	struct rte_event_port_conf *port_conf = conf_arg;
+	uint8_t disable_impl_rel;
 	int ret;
 
 	RTE_EVENT_ETH_TX_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
@@ -986,8 +993,11 @@ rte_event_eth_tx_adapter_create_ext(uint8_t id, uint8_t dev_id,
 		return ret;
 	}
 
+	disable_impl_rel = !!(port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+
 	rte_eventdev_trace_eth_tx_adapter_create(id, dev_id, conf_cb, conf_arg,
-		ret);
+		disable_impl_rel, ret);
 	txa_dev_id_array[id] = dev_id;
 	return 0;
 }
diff --git a/lib/librte_eventdev/rte_eventdev.c b/lib/librte_eventdev/rte_eventdev.c
index 4955ab1a0..67c9c8357 100644
--- a/lib/librte_eventdev/rte_eventdev.c
+++ b/lib/librte_eventdev/rte_eventdev.c
@@ -736,6 +736,7 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
 {
 	struct rte_eventdev *dev;
 	struct rte_event_port_conf def_conf;
+	uint8_t disable_impl_rel;
 	int diag;
 
 	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
@@ -812,7 +813,12 @@ rte_event_port_setup(uint8_t dev_id, uint8_t port_id,
 	if (!diag)
 		diag = rte_event_port_unlink(dev_id, port_id, NULL, 0);
 
-	rte_eventdev_trace_port_setup(dev_id, port_id, port_conf, diag);
+	disable_impl_rel = !!(port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+
+	rte_eventdev_trace_port_setup(dev_id, port_id, port_conf,
+				      disable_impl_rel, diag);
+
 	if (diag < 0)
 		return diag;
 
@@ -1422,10 +1428,8 @@ rte_event_pmd_allocate(const char *name, int socket_id)
 		eventdev->data = eventdev_data;
 
 		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
-
 			strlcpy(eventdev->data->name, name,
 				RTE_EVENTDEV_NAME_MAX_LEN);
-
 			eventdev->data->dev_id = dev_id;
 			eventdev->data->socket_id = socket_id;
 			eventdev->data->dev_started = 0;
diff --git a/lib/librte_eventdev/rte_eventdev_trace.h b/lib/librte_eventdev/rte_eventdev_trace.h
index 4de6341ca..f56063976 100644
--- a/lib/librte_eventdev/rte_eventdev_trace.h
+++ b/lib/librte_eventdev/rte_eventdev_trace.h
@@ -53,13 +53,14 @@ RTE_TRACE_POINT(
 RTE_TRACE_POINT(
 	rte_eventdev_trace_port_setup,
 	RTE_TRACE_POINT_ARGS(uint8_t dev_id, uint8_t port_id,
-		const struct rte_event_port_conf *port_conf, int rc),
+		const struct rte_event_port_conf *port_conf,
+		uint8_t disable_impl_rel, int rc),
 	rte_trace_point_emit_u8(dev_id);
 	rte_trace_point_emit_u8(port_id);
 	rte_trace_point_emit_i32(port_conf->new_event_threshold);
 	rte_trace_point_emit_u16(port_conf->dequeue_depth);
 	rte_trace_point_emit_u16(port_conf->enqueue_depth);
-	rte_trace_point_emit_u8(port_conf->disable_implicit_release);
+	rte_trace_point_emit_u8(disable_impl_rel);
 	rte_trace_point_emit_int(rc);
 )
 
@@ -159,13 +160,14 @@ RTE_TRACE_POINT(
 RTE_TRACE_POINT(
 	rte_eventdev_trace_eth_tx_adapter_create,
 	RTE_TRACE_POINT_ARGS(uint8_t adptr_id, uint8_t dev_id, void *conf_cb,
-		struct rte_event_port_conf *port_conf, int rc),
+		struct rte_event_port_conf *port_conf, uint8_t disable_impl_rel,
+		int rc),
 	rte_trace_point_emit_u8(adptr_id);
 	rte_trace_point_emit_u8(dev_id);
 	rte_trace_point_emit_i32(port_conf->new_event_threshold);
 	rte_trace_point_emit_u16(port_conf->dequeue_depth);
 	rte_trace_point_emit_u16(port_conf->enqueue_depth);
-	rte_trace_point_emit_u8(port_conf->disable_implicit_release);
+	rte_trace_point_emit_u8(disable_impl_rel);
 	rte_trace_point_emit_ptr(conf_cb);
 	rte_trace_point_emit_int(rc);
 )
@@ -249,7 +251,8 @@ RTE_TRACE_POINT(
 RTE_TRACE_POINT(
 	rte_eventdev_trace_crypto_adapter_create,
 	RTE_TRACE_POINT_ARGS(uint8_t adptr_id, uint8_t dev_id, void *adapter,
-		struct rte_event_port_conf *port_conf, uint8_t mode),
+		struct rte_event_port_conf *port_conf, uint8_t disable_impl_rel,
+		uint8_t mode),
 	rte_trace_point_emit_u8(adptr_id);
 	rte_trace_point_emit_u8(dev_id);
 	rte_trace_point_emit_ptr(adapter);
@@ -257,7 +260,7 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(port_conf->new_event_threshold);
 	rte_trace_point_emit_u16(port_conf->dequeue_depth);
 	rte_trace_point_emit_u16(port_conf->enqueue_depth);
-	rte_trace_point_emit_u8(port_conf->disable_implicit_release);
+	rte_trace_point_emit_u8(disable_impl_rel);
 )
 
 RTE_TRACE_POINT(
-- 
2.13.6

[dpdk-dev] [PATCH 03/27] event/dlb: add shared code version 10.7.9

[McDaniel, Timothy]

The DLB shared code is auto generated by Intel, and is being committed
here so that it can be built in the DPDK environment. The shared code
should not be modified. The shared code must be present in order to
successfully build the DLB PMD.

Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  360 +
 drivers/event/dlb/pf/base/dlb_mbox.h         |  645 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  350 +
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  449 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2646 +++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 9699 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     | 1625 +++++
 drivers/event/dlb/pf/base/dlb_user.h         | 1084 +++
 10 files changed, 17020 insertions(+)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_mbox.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_user.h

diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 000000000..d56590e64
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,360 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+#define DLB_VF_DEV_ID 0x270C
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_VF_NUM_NON_CQ_INTERRUPT_VECTORS 1
+#define DLB_VF_NUM_CQ_INTERRUPT_VECTORS 31
+#define DLB_VF_BASE_CQ_VECTOR_ID 0
+#define DLB_VF_LAST_CQ_VECTOR_ID 30
+#define DLB_VF_MBOX_VECTOR_ID 31
+#define DLB_VF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_VF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_VF_NUM_CQ_INTERRUPT_VECTORS)
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+/* DLB ALARM interrupts */
+#define DLB_INT_ALARM 0
+/* VF to PF Mailbox Service Request */
+#define DLB_INT_VF_TO_PF_MBOX 1
+/* HCW Ingress Errors */
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_resource_id {
+	u32 phys_id;
+	u32 virt_id;
+	u8 vf_owned;
+	u8 vf_id;
+};
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	struct dlb_resource_id id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports; /* (PF only) */
+	u8 locked; /* (VF only) */
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A VF's available resources list. These are VF-owned unconfigured
+ *	resources not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a VF or domain is created or destroyed,
+ * or when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_function_resources vf[DLB_MAX_NUM_VFS];
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_mbox.h b/drivers/event/dlb/pf/base/dlb_mbox.h
new file mode 100644
index 000000000..e195526a2
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_mbox.h
@@ -0,0 +1,645 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_BASE_DLB_MBOX_H
+#define __DLB_BASE_DLB_MBOX_H
+
+#include "dlb_regs.h"
+#include "dlb_osdep_types.h"
+
+#define DLB_MBOX_INTERFACE_VERSION 1
+
+/* The PF uses its PF->VF mailbox to send responses to VF requests, as well as
+ * to send requests of its own (e.g. notifying a VF of an impending FLR).
+ * To avoid communication race conditions, e.g. the PF sends a response and then
+ * sends a request before the VF reads the response, the PF->VF mailbox is
+ * divided into two sections:
+ * - Bytes 0-47: PF responses
+ * - Bytes 48-63: PF requests
+ *
+ * Partitioning the PF->VF mailbox allows responses and requests to occupy the
+ * mailbox simultaneously.
+ */
+#define DLB_PF2VF_RESP_BYTES 48
+#define DLB_PF2VF_RESP_BASE 0
+#define DLB_PF2VF_RESP_BASE_WORD (DLB_PF2VF_RESP_BASE / 4)
+
+#define DLB_PF2VF_REQ_BYTES \
+	(DLB_FUNC_PF_PF2VF_MAILBOX_BYTES - DLB_PF2VF_RESP_BYTES)
+#define DLB_PF2VF_REQ_BASE DLB_PF2VF_RESP_BYTES
+#define DLB_PF2VF_REQ_BASE_WORD (DLB_PF2VF_REQ_BASE / 4)
+
+/* Similarly, the VF->PF mailbox is divided into two sections:
+ * - Bytes 0-239: VF requests
+ * - Bytes 240-255: VF responses
+ */
+#define DLB_VF2PF_REQ_BYTES 240
+#define DLB_VF2PF_REQ_BASE 0
+#define DLB_VF2PF_REQ_BASE_WORD (DLB_VF2PF_REQ_BASE / 4)
+
+#define DLB_VF2PF_RESP_BYTES \
+	(DLB_FUNC_VF_VF2PF_MAILBOX_BYTES - DLB_VF2PF_REQ_BYTES)
+#define DLB_VF2PF_RESP_BASE DLB_VF2PF_REQ_BYTES
+#define DLB_VF2PF_RESP_BASE_WORD (DLB_VF2PF_RESP_BASE / 4)
+
+/* VF-initiated commands */
+enum dlb_mbox_cmd_type {
+	DLB_MBOX_CMD_REGISTER,
+	DLB_MBOX_CMD_UNREGISTER,
+	DLB_MBOX_CMD_GET_NUM_RESOURCES,
+	DLB_MBOX_CMD_CREATE_SCHED_DOMAIN,
+	DLB_MBOX_CMD_RESET_SCHED_DOMAIN,
+	DLB_MBOX_CMD_CREATE_LDB_POOL,
+	DLB_MBOX_CMD_CREATE_DIR_POOL,
+	DLB_MBOX_CMD_CREATE_LDB_QUEUE,
+	DLB_MBOX_CMD_CREATE_DIR_QUEUE,
+	DLB_MBOX_CMD_CREATE_LDB_PORT,
+	DLB_MBOX_CMD_CREATE_DIR_PORT,
+	DLB_MBOX_CMD_ENABLE_LDB_PORT,
+	DLB_MBOX_CMD_DISABLE_LDB_PORT,
+	DLB_MBOX_CMD_ENABLE_DIR_PORT,
+	DLB_MBOX_CMD_DISABLE_DIR_PORT,
+	DLB_MBOX_CMD_LDB_PORT_OWNED_BY_DOMAIN,
+	DLB_MBOX_CMD_DIR_PORT_OWNED_BY_DOMAIN,
+	DLB_MBOX_CMD_MAP_QID,
+	DLB_MBOX_CMD_UNMAP_QID,
+	DLB_MBOX_CMD_START_DOMAIN,
+	DLB_MBOX_CMD_ENABLE_LDB_PORT_INTR,
+	DLB_MBOX_CMD_ENABLE_DIR_PORT_INTR,
+	DLB_MBOX_CMD_ARM_CQ_INTR,
+	DLB_MBOX_CMD_GET_NUM_USED_RESOURCES,
+	DLB_MBOX_CMD_INIT_CQ_SCHED_COUNT,
+	DLB_MBOX_CMD_COLLECT_CQ_SCHED_COUNT,
+	DLB_MBOX_CMD_ACK_VF_FLR_DONE,
+	DLB_MBOX_CMD_GET_SN_ALLOCATION,
+	DLB_MBOX_CMD_GET_LDB_QUEUE_DEPTH,
+	DLB_MBOX_CMD_GET_DIR_QUEUE_DEPTH,
+	DLB_MBOX_CMD_PENDING_PORT_UNMAPS,
+	DLB_MBOX_CMD_QUERY_CQ_POLL_MODE,
+	DLB_MBOX_CMD_GET_SN_OCCUPANCY,
+
+	/* NUM_QE_CMD_TYPES must be last */
+	NUM_DLB_MBOX_CMD_TYPES,
+};
+
+static const char dlb_mbox_cmd_type_strings[][128] = {
+	"DLB_MBOX_CMD_REGISTER",
+	"DLB_MBOX_CMD_UNREGISTER",
+	"DLB_MBOX_CMD_GET_NUM_RESOURCES",
+	"DLB_MBOX_CMD_CREATE_SCHED_DOMAIN",
+	"DLB_MBOX_CMD_RESET_SCHED_DOMAIN",
+	"DLB_MBOX_CMD_CREATE_LDB_POOL",
+	"DLB_MBOX_CMD_CREATE_DIR_POOL",
+	"DLB_MBOX_CMD_CREATE_LDB_QUEUE",
+	"DLB_MBOX_CMD_CREATE_DIR_QUEUE",
+	"DLB_MBOX_CMD_CREATE_LDB_PORT",
+	"DLB_MBOX_CMD_CREATE_DIR_PORT",
+	"DLB_MBOX_CMD_ENABLE_LDB_PORT",
+	"DLB_MBOX_CMD_DISABLE_LDB_PORT",
+	"DLB_MBOX_CMD_ENABLE_DIR_PORT",
+	"DLB_MBOX_CMD_DISABLE_DIR_PORT",
+	"DLB_MBOX_CMD_LDB_PORT_OWNED_BY_DOMAIN",
+	"DLB_MBOX_CMD_DIR_PORT_OWNED_BY_DOMAIN",
+	"DLB_MBOX_CMD_MAP_QID",
+	"DLB_MBOX_CMD_UNMAP_QID",
+	"DLB_MBOX_CMD_START_DOMAIN",
+	"DLB_MBOX_CMD_ENABLE_LDB_PORT_INTR",
+	"DLB_MBOX_CMD_ENABLE_DIR_PORT_INTR",
+	"DLB_MBOX_CMD_ARM_CQ_INTR",
+	"DLB_MBOX_CMD_GET_NUM_USED_RESOURCES",
+	"DLB_MBOX_CMD_INIT_CQ_SCHED_COUNT",
+	"DLB_MBOX_CMD_COLLECT_CQ_SCHED_COUNT",
+	"DLB_MBOX_CMD_ACK_VF_FLR_DONE",
+	"DLB_MBOX_CMD_GET_SN_ALLOCATION",
+	"DLB_MBOX_CMD_GET_LDB_QUEUE_DEPTH",
+	"DLB_MBOX_CMD_GET_DIR_QUEUE_DEPTH",
+	"DLB_MBOX_CMD_PENDING_PORT_UNMAPS",
+	"DLB_MBOX_CMD_QUERY_CQ_POLL_MODE",
+	"DLB_MBOX_CMD_GET_SN_OCCUPANCY",
+};
+
+/* PF-initiated commands */
+enum dlb_mbox_vf_cmd_type {
+	DLB_MBOX_VF_CMD_DOMAIN_ALERT,
+	DLB_MBOX_VF_CMD_NOTIFICATION,
+	DLB_MBOX_VF_CMD_IN_USE,
+
+	/* NUM_DLB_MBOX_VF_CMD_TYPES must be last */
+	NUM_DLB_MBOX_VF_CMD_TYPES,
+};
+
+static const char dlb_mbox_vf_cmd_type_strings[][128] = {
+	"DLB_MBOX_VF_CMD_DOMAIN_ALERT",
+	"DLB_MBOX_VF_CMD_NOTIFICATION",
+	"DLB_MBOX_VF_CMD_IN_USE",
+};
+
+#define DLB_MBOX_CMD_TYPE(hdr) \
+	(((struct dlb_mbox_req_hdr *)hdr)->type)
+#define DLB_MBOX_CMD_STRING(hdr) \
+	dlb_mbox_cmd_type_strings[DLB_MBOX_CMD_TYPE(hdr)]
+
+enum dlb_mbox_status_type {
+	DLB_MBOX_ST_SUCCESS,
+	DLB_MBOX_ST_INVALID_CMD_TYPE,
+	DLB_MBOX_ST_VERSION_MISMATCH,
+	DLB_MBOX_ST_EXPECTED_PHASE_ONE,
+	DLB_MBOX_ST_EXPECTED_PHASE_TWO,
+	DLB_MBOX_ST_INVALID_OWNER_VF,
+};
+
+static const char dlb_mbox_status_type_strings[][128] = {
+	"DLB_MBOX_ST_SUCCESS",
+	"DLB_MBOX_ST_INVALID_CMD_TYPE",
+	"DLB_MBOX_ST_VERSION_MISMATCH",
+	"DLB_MBOX_ST_EXPECTED_PHASE_ONE",
+	"DLB_MBOX_ST_EXPECTED_PHASE_TWO",
+	"DLB_MBOX_ST_INVALID_OWNER_VF",
+};
+
+#define DLB_MBOX_ST_TYPE(hdr) \
+	(((struct dlb_mbox_resp_hdr *)hdr)->status)
+#define DLB_MBOX_ST_STRING(hdr) \
+	dlb_mbox_status_type_strings[DLB_MBOX_ST_TYPE(hdr)]
+
+/* This structure is always the first field in a request structure */
+struct dlb_mbox_req_hdr {
+	u32 type;
+};
+
+/* This structure is always the first field in a response structure */
+struct dlb_mbox_resp_hdr {
+	u32 status;
+};
+
+struct dlb_mbox_register_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 min_interface_version;
+	u16 max_interface_version;
+};
+
+struct dlb_mbox_register_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 interface_version;
+	u8 pf_id;
+	u8 vf_id;
+	u8 is_auxiliary_vf;
+	u8 primary_vf_id;
+	u32 padding;
+};
+
+struct dlb_mbox_unregister_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_unregister_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_get_num_resources_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_get_num_resources_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u16 num_sched_domains;
+	u16 num_ldb_queues;
+	u16 num_ldb_ports;
+	u16 num_dir_ports;
+	u16 padding0;
+	u8 num_ldb_credit_pools;
+	u8 num_dir_credit_pools;
+	u32 num_atomic_inflights;
+	u32 max_contiguous_atomic_inflights;
+	u32 num_hist_list_entries;
+	u32 max_contiguous_hist_list_entries;
+	u16 num_ldb_credits;
+	u16 max_contiguous_ldb_credits;
+	u16 num_dir_credits;
+	u16 max_contiguous_dir_credits;
+	u32 padding1;
+};
+
+struct dlb_mbox_create_sched_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 num_ldb_queues;
+	u32 num_ldb_ports;
+	u32 num_dir_ports;
+	u32 num_atomic_inflights;
+	u32 num_hist_list_entries;
+	u32 num_ldb_credits;
+	u32 num_dir_credits;
+	u32 num_ldb_credit_pools;
+	u32 num_dir_credit_pools;
+};
+
+struct dlb_mbox_create_sched_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_reset_sched_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 id;
+};
+
+struct dlb_mbox_reset_sched_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+};
+
+struct dlb_mbox_create_credit_pool_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 num_credits;
+	u32 padding;
+};
+
+struct dlb_mbox_create_credit_pool_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_ldb_queue_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 num_sequence_numbers;
+	u32 num_qid_inflights;
+	u32 num_atomic_inflights;
+	u32 padding;
+};
+
+struct dlb_mbox_create_ldb_queue_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_dir_queue_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding0;
+};
+
+struct dlb_mbox_create_dir_queue_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 ldb_credit_pool_id;
+	u32 dir_credit_pool_id;
+	u64 pop_count_address;
+	u16 ldb_credit_high_watermark;
+	u16 ldb_credit_low_watermark;
+	u16 ldb_credit_quantum;
+	u16 dir_credit_high_watermark;
+	u16 dir_credit_low_watermark;
+	u16 dir_credit_quantum;
+	u32 padding0;
+	u16 cq_depth;
+	u16 cq_history_list_size;
+	u32 padding1;
+	u64 cq_base_address;
+	u64 nq_base_address;
+	u32 nq_size;
+	u32 padding2;
+};
+
+struct dlb_mbox_create_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 ldb_credit_pool_id;
+	u32 dir_credit_pool_id;
+	u64 pop_count_address;
+	u16 ldb_credit_high_watermark;
+	u16 ldb_credit_low_watermark;
+	u16 ldb_credit_quantum;
+	u16 dir_credit_high_watermark;
+	u16 dir_credit_low_watermark;
+	u16 dir_credit_quantum;
+	u16 cq_depth;
+	u16 padding0;
+	u64 cq_base_address;
+	s32 queue_id;
+	u32 padding1;
+};
+
+struct dlb_mbox_create_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_enable_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_ldb_port_owned_by_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_ldb_port_owned_by_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	s32 owned;
+};
+
+struct dlb_mbox_dir_port_owned_by_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_dir_port_owned_by_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	s32 owned;
+};
+
+struct dlb_mbox_map_qid_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 qid;
+	u32 priority;
+	u32 padding0;
+};
+
+struct dlb_mbox_map_qid_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_unmap_qid_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 qid;
+};
+
+struct dlb_mbox_unmap_qid_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_start_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+};
+
+struct dlb_mbox_start_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_ldb_port_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 port_id;
+	u16 thresh;
+	u16 vector;
+	u16 owner_vf;
+	u16 reserved[2];
+};
+
+struct dlb_mbox_enable_ldb_port_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+struct dlb_mbox_enable_dir_port_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 port_id;
+	u16 thresh;
+	u16 vector;
+	u16 owner_vf;
+	u16 reserved[2];
+};
+
+struct dlb_mbox_enable_dir_port_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+struct dlb_mbox_arm_cq_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 is_ldb;
+};
+
+struct dlb_mbox_arm_cq_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+/* The alert_id and aux_alert_data follows the format of the alerts defined in
+ * dlb_types.h. The alert id contains an enum dlb_domain_alert_id value, and
+ * the aux_alert_data value varies depending on the alert.
+ */
+struct dlb_mbox_vf_alert_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 alert_id;
+	u32 aux_alert_data;
+};
+
+enum dlb_mbox_vf_notification_type {
+	DLB_MBOX_VF_NOTIFICATION_PRE_RESET,
+	DLB_MBOX_VF_NOTIFICATION_POST_RESET,
+
+	/* NUM_DLB_MBOX_VF_NOTIFICATION_TYPES must be last */
+	NUM_DLB_MBOX_VF_NOTIFICATION_TYPES,
+};
+
+struct dlb_mbox_vf_notification_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 notification;
+};
+
+struct dlb_mbox_vf_in_use_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_vf_in_use_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 in_use;
+};
+
+struct dlb_mbox_ack_vf_flr_done_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_ack_vf_flr_done_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_get_sn_allocation_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 group_id;
+};
+
+struct dlb_mbox_get_sn_allocation_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 num;
+};
+
+struct dlb_mbox_get_ldb_queue_depth_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 queue_id;
+	u32 padding;
+};
+
+struct dlb_mbox_get_ldb_queue_depth_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 depth;
+};
+
+struct dlb_mbox_get_dir_queue_depth_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 queue_id;
+	u32 padding;
+};
+
+struct dlb_mbox_get_dir_queue_depth_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 depth;
+};
+
+struct dlb_mbox_pending_port_unmaps_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_pending_port_unmaps_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 num;
+};
+
+struct dlb_mbox_query_cq_poll_mode_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_query_cq_poll_mode_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 mode;
+};
+
+struct dlb_mbox_get_sn_occupancy_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 group_id;
+};
+
+struct dlb_mbox_get_sn_occupancy_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 num;
+};
+
+#endif /* __DLB_BASE_DLB_MBOX_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 000000000..8b1d22bbb
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,350 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+#define READ_ONCE(x) (x)
+#define WRITE_ONCE(x, y) ((x) = (y))
+
+#define OS_READ_ONCE(x) READ_ONCE(x)
+#define OS_WRITE_ONCE(x, y) WRITE_ONCE(x, y)
+
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+/**
+ * os_curtime_s() - get the current time (in seconds)
+ * @usecs: delay duration.
+ */
+static inline unsigned long os_curtime_s(void)
+{
+	struct timespec tv;
+
+	clock_gettime(CLOCK_MONOTONIC, &tv);
+
+	return (unsigned long)tv.tv_sec;
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+/**
+ * os_enqueue_four_hcws() - enqueue four HCWs to DLB
+ * @hw: dlb_hw handle for a particular device.
+ * @hcw: pointer to the 64B-aligned contiguous HCW memory
+ * @addr: producer port address
+ */
+static inline void os_enqueue_four_hcws(struct dlb_hw *hw,
+					struct dlb_hcw *hcw,
+					void *addr)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	dlb_dev->enqueue_four(hcw, addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+#define DLB_ERR(dev, fmt, args...) \
+	RTE_LOG(ERR, PMD, "%s() line %u: " fmt "\n",  \
+			__func__, __LINE__, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	RTE_LOG(INFO, PMD, "%s() line %u: " fmt "\n", \
+			__func__, __LINE__, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	RTE_LOG(DEBUG, PMD, "%s() line %u: " fmt "\n", \
+			__func__, __LINE__, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function doesn't deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = pthread_create(&complete_queue_map_unmap_thread,
+			     NULL,
+			     dlb_complete_queue_map_unmap,
+			     dlb_dev);
+
+	/* PFPMD_FIXME - this function should be allowed to return an error */
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map /unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	rte_panic("internal_error: %s should never be called for DLB PF PMD\n",
+		  __func__);
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+#include <cpuid.h>
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 000000000..2c95796f5
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,449 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (!bitmap || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (!bm)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (!mem) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (!bm->map) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (!bitmap)
+		rte_panic("NULL dlb_bitmap in %s\n", __func__);
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (!bitmap || !bitmap->map || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	/* TODO - optimize */
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	/* TODO - optimize */
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (!dest || !dest->map ||
+	    !src1 || !src1->map ||
+	    !src2 || !src2->map)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 000000000..a53b3626e
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 000000000..2e9d7d8d0
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 000000000..3b0be239a
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2646 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_FUNC_PF_VF2PF_MAILBOX_BYTES 256
+#define DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, x) \
+	(0x1000 + 0x4 * (x) + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_MAILBOX_RST 0x0
+union dlb_func_pf_vf2pf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_MAILBOX_ISR(vf_id) \
+	(0x1f00 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_MAILBOX_ISR_RST 0x0
+union dlb_func_pf_vf2pf_mailbox_isr {
+	struct {
+		u32 vf_isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_FLR_ISR(vf_id) \
+	(0x1f04 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_FLR_ISR_RST 0x0
+union dlb_func_pf_vf2pf_flr_isr {
+	struct {
+		u32 vf_isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_ISR_PEND(vf_id) \
+	(0x1f10 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_ISR_PEND_RST 0x0
+union dlb_func_pf_vf2pf_isr_pend {
+	struct {
+		u32 isr_pend : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_PF2VF_MAILBOX_BYTES 64
+#define DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, x) \
+	(0x2000 + 0x4 * (x) + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_PF2VF_MAILBOX_RST 0x0
+union dlb_func_pf_pf2vf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_PF2VF_MAILBOX_ISR(vf_id) \
+	(0x2f00 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_PF2VF_MAILBOX_ISR_RST 0x0
+union dlb_func_pf_pf2vf_mailbox_isr {
+	struct {
+		u32 isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF_RESET_IN_PROGRESS(vf_id) \
+	(0x3000 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF_RESET_IN_PROGRESS_RST 0xffff
+union dlb_func_pf_vf_reset_in_progress {
+	struct {
+		u32 reset_in_progress : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VPP_V(x) \
+	(0xf00 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VPP_V_RST 0x0
+union dlb_sys_vf_ldb_vpp_v {
+	struct {
+		u32 vpp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VPP2PP(x) \
+	(0xf08 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VPP2PP_RST 0x0
+union dlb_sys_vf_ldb_vpp2pp {
+	struct {
+		u32 pp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VPP_V(x) \
+	(0xf10 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VPP_V_RST 0x0
+union dlb_sys_vf_dir_vpp_v {
+	struct {
+		u32 vpp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VPP2PP(x) \
+	(0xf18 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VPP2PP_RST 0x0
+union dlb_sys_vf_dir_vpp2pp {
+	struct {
+		u32 pp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VQID_V(x) \
+	(0xf20 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VQID_V_RST 0x0
+union dlb_sys_vf_ldb_vqid_v {
+	struct {
+		u32 vqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VQID2QID(x) \
+	(0xf28 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VQID2QID_RST 0x0
+union dlb_sys_vf_ldb_vqid2qid {
+	struct {
+		u32 qid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID2VQID(x) \
+	(0xf2c + (x) * 0x1000)
+#define DLB_SYS_LDB_QID2VQID_RST 0x0
+union dlb_sys_ldb_qid2vqid {
+	struct {
+		u32 vqid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VQID_V(x) \
+	(0xf30 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VQID_V_RST 0x0
+union dlb_sys_vf_dir_vqid_v {
+	struct {
+		u32 vqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VQID2QID(x) \
+	(0xf38 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VQID2QID_RST 0x0
+union dlb_sys_vf_dir_vqid2qid {
+	struct {
+		u32 qid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND2(x) \
+	(0x8000018 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND2_RST 0x0
+union dlb_sys_alarm_vf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND1(x) \
+	(0x8000014 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND1_RST 0x0
+union dlb_sys_alarm_vf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND0(x) \
+	(0x8000010 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND0_RST 0x0
+union dlb_sys_alarm_vf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_FUNC_VF_BAR_DSBL(x) \
+	(0x310 + (x) * 0x4)
+#define DLB_SYS_FUNC_VF_BAR_DSBL_RST 0x0
+union dlb_sys_func_vf_bar_dsbl {
+	struct {
+		u32 func_vf_bar_dis : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF2PF_MAILBOX_BYTES 256
+#define DLB_FUNC_VF_VF2PF_MAILBOX(x) \
+	(0x1000 + (x) * 0x4)
+#define DLB_FUNC_VF_VF2PF_MAILBOX_RST 0x0
+union dlb_func_vf_vf2pf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF2PF_MAILBOX_ISR 0x1f00
+#define DLB_FUNC_VF_VF2PF_MAILBOX_ISR_RST 0x0
+union dlb_func_vf_vf2pf_mailbox_isr {
+	struct {
+		u32 isr : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_PF2VF_MAILBOX_BYTES 64
+#define DLB_FUNC_VF_PF2VF_MAILBOX(x) \
+	(0x2000 + (x) * 0x4)
+#define DLB_FUNC_VF_PF2VF_MAILBOX_RST 0x0
+union dlb_func_vf_pf2vf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_PF2VF_MAILBOX_ISR 0x2f00
+#define DLB_FUNC_VF_PF2VF_MAILBOX_ISR_RST 0x0
+union dlb_func_vf_pf2vf_mailbox_isr {
+	struct {
+		u32 pf_isr : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_MSI_ISR_PEND 0x2f10
+#define DLB_FUNC_VF_VF_MSI_ISR_PEND_RST 0x0
+union dlb_func_vf_vf_msi_isr_pend {
+	struct {
+		u32 isr_pend : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_RESET_IN_PROGRESS 0x3000
+#define DLB_FUNC_VF_VF_RESET_IN_PROGRESS_RST 0x1
+union dlb_func_vf_vf_reset_in_progress {
+	struct {
+		u32 reset_in_progress : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_MSI_ISR 0x4000
+#define DLB_FUNC_VF_VF_MSI_ISR_RST 0x0
+union dlb_func_vf_vf_msi_isr {
+	struct {
+		u32 vf_msi_isr : 32;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 000000000..cef81b8b4
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,9699 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+/* Copyright(c) 2016-2020 Intel Corporation */
+
+#include "dlb_hw_types.h"
+#include "dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+#include "dlb_mbox.h"
+
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
+
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
+
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
+
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
+
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
+
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
+
+/* The PF driver cannot assume that a register write will affect subsequent HCW
+ * writes. To ensure a write completes, the driver must read back a CSR. This
+ * function only need be called for configuration that can occur after the
+ * domain has started; prior to starting, applications can't send HCWs.
+ */
+static inline void dlb_flush_csr(struct dlb_hw *hw)
+{
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		memset(&hw->vf[i], 0, sizeof(hw->vf[i]));
+		dlb_init_fn_rsrc_lists(&hw->vf[i]);
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_hist_list_entries,
+				     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_qed_freelist_entries,
+				     DLB_MAX_NUM_LDB_CREDITS))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_dqed_freelist_entries,
+				     DLB_MAX_NUM_DIR_CREDITS))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_aqed_freelist_entries,
+				     DLB_MAX_NUM_AQOS_ENTRIES))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_hist_list_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_qed_freelist_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_dqed_freelist_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_aqed_freelist_entries))
+			return -1;
+	}
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		hw->domains[i].id.phys_id = i;
+		hw->domains[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++) {
+		hw->rsrcs.ldb_queues[i].id.phys_id = i;
+		hw->rsrcs.ldb_queues[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++) {
+		hw->rsrcs.ldb_ports[i].id.phys_id = i;
+		hw->rsrcs.ldb_ports[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++) {
+		hw->rsrcs.dir_pq_pairs[i].id.phys_id = i;
+		hw->rsrcs.dir_pq_pairs[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++) {
+		hw->rsrcs.ldb_credit_pools[i].id.phys_id = i;
+		hw->rsrcs.ldb_credit_pools[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++) {
+		hw->rsrcs.dir_credit_pools[i].id.phys_id = i;
+		hw->rsrcs.dir_credit_pools[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	int i;
+
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		dlb_bitmap_free(hw->vf[i].avail_hist_list_entries);
+		dlb_bitmap_free(hw->vf[i].avail_qed_freelist_entries);
+		dlb_bitmap_free(hw->vf[i].avail_dqed_freelist_entries);
+		dlb_bitmap_free(hw->vf[i].avail_aqed_freelist_entries);
+	}
+}
+
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw,
+						 u32 id,
+						 bool vf_request,
+						 unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	if (!vf_request)
+		return &hw->domains[id];
+
+	rsrcs = &hw->vf[vf_id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter)
+		if (domain->id.virt_id == id)
+			return domain;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id,
+			bool vf_request,
+			struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if ((!vf_request && pool->id.phys_id == id) ||
+		    (vf_request && pool->id.virt_id == id))
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id,
+			bool vf_request,
+			struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if ((!vf_request && pool->id.phys_id == id) ||
+		    (vf_request && pool->id.virt_id == id))
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *dlb_get_ldb_port_from_id(struct dlb_hw *hw,
+						     u32 id,
+						     bool vf_request,
+						     unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.ldb_ports[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter2)
+			if (port->id.virt_id == id)
+				return port;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter1)
+		if (port->id.virt_id == id)
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id,
+			     bool vf_request,
+			     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *dlb_get_domain_ldb_port(u32 id,
+						    bool vf_request,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *dlb_get_dir_pq_from_id(struct dlb_hw *hw,
+						      u32 id,
+						      bool vf_request,
+						      unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.dir_pq_pairs[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter2)
+			if (port->id.virt_id == id)
+				return port;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_pq_pairs, port, iter1)
+		if (port->id.virt_id == id)
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id,
+			   bool vf_request,
+			   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *dlb_get_domain_dir_pq(u32 id,
+						     bool vf_request,
+						     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id,
+						       bool vf_request,
+						       unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.ldb_queues[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
+			if (queue->id.virt_id == id)
+				return queue;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, queue, iter1)
+		if (queue->id.virt_id == id)
+			return queue;
+
+	return NULL;
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      bool vf_request,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if ((!vf_request && queue->id.phys_id == id) ||
+		    (vf_request && queue->id.virt_id == id))
+			return queue;
+
+	return NULL;
+}
+
+#define DLB_XFER_LL_RSRC(dst, src, num, type_t, name) ({		    \
+	struct dlb_list_entry *it1 __attribute__((unused));		    \
+	struct dlb_list_entry *it2 __attribute__((unused));		    \
+	struct dlb_function_resources *_src = src;			    \
+	struct dlb_function_resources *_dst = dst;			    \
+	type_t *ptr, *tmp __attribute__((unused));			    \
+	unsigned int i = 0;						    \
+									    \
+	DLB_FUNC_LIST_FOR_SAFE(_src->avail_##name##s, ptr, tmp, it1, it2) { \
+		if (i++ == (num))					    \
+			break;						    \
+									    \
+		dlb_list_del(&_src->avail_##name##s, &ptr->func_list);	    \
+		dlb_list_add(&_dst->avail_##name##s,  &ptr->func_list);     \
+		_src->num_avail_##name##s--;				    \
+		_dst->num_avail_##name##s++;				    \
+	}								    \
+})
+
+#define DLB_VF_ID_CLEAR(head, type_t) ({   \
+	struct dlb_list_entry *iter __attribute__((unused)); \
+	type_t *var;					     \
+							     \
+	DLB_FUNC_LIST_FOR(head, var, iter)		     \
+		var->id.vf_owned = false;		     \
+})
+
+int dlb_update_vf_sched_domains(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_function_resources *src, *dst;
+	struct dlb_domain *domain;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_domains;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_domains, struct dlb_domain);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_domain, domain);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_domains) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_domain, domain);
+
+	/* Set the domains' VF backpointer */
+	DLB_FUNC_LIST_FOR(dst->avail_domains, domain, iter)
+		domain->parent_func = dst;
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_queues(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_queues;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_queues, struct dlb_ldb_queue);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_ldb_queue, ldb_queue);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_queues) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_ldb_queue, ldb_queue);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_ports(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_ports;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_ports, struct dlb_ldb_port);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_ldb_port, ldb_port);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_ports) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_ldb_port, ldb_port);
+
+	return ret;
+}
+
+int dlb_update_vf_dir_ports(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_dir_pq_pairs;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_dir_pq_pairs, struct dlb_dir_pq_pair);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_dir_pq_pair, dir_pq_pair);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_dir_pq_pairs) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_dir_pq_pair, dir_pq_pair);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_credit_pools;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_credit_pools, struct dlb_credit_pool);
+
+	DLB_XFER_LL_RSRC(src,
+			 dst,
+			 orig,
+			 struct dlb_credit_pool,
+			 ldb_credit_pool);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_credit_pools) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst,
+			 src,
+			 num,
+			 struct dlb_credit_pool,
+			 ldb_credit_pool);
+
+	return ret;
+}
+
+int dlb_update_vf_dir_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_dir_credit_pools;
+
+	/* Detach the VF's current resources before checking if enough are
+	 * available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_dir_credit_pools, struct dlb_credit_pool);
+
+	DLB_XFER_LL_RSRC(src,
+			 dst,
+			 orig,
+			 struct dlb_credit_pool,
+			 dir_credit_pool);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_dir_credit_pools) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst,
+			 src,
+			 num,
+			 struct dlb_credit_pool,
+			 dir_credit_pool);
+
+	return ret;
+}
+
+static int dlb_transfer_bitmap_resources(struct dlb_bitmap *src,
+					 struct dlb_bitmap *dst,
+					 u32 num)
+{
+	int orig, ret, base;
+
+	/* Validate bitmaps before use */
+	if (dlb_bitmap_count(dst) < 0 || dlb_bitmap_count(src) < 0)
+		return -EINVAL;
+
+	/* Reassign the dest's bitmap entries to the source's before checking
+	 * if a contiguous chunk of size 'num' is available. The reassignment
+	 * may be necessary to create a sufficiently large contiguous chunk.
+	 */
+	orig = dlb_bitmap_count(dst);
+
+	dlb_bitmap_or(src, src, dst);
+
+	dlb_bitmap_zero(dst);
+
+	/* Are there enough available resources to satisfy the request? */
+	base = dlb_bitmap_find_set_bit_range(src, num);
+
+	if (base == -ENOENT) {
+		num = orig;
+		base = dlb_bitmap_find_set_bit_range(src, num);
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	dlb_bitmap_set_range(dst, base, num);
+
+	dlb_bitmap_clear_range(src, base, num);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_credits(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_qed_freelist_entries,
+					     dst->avail_qed_freelist_entries,
+					     num);
+}
+
+int dlb_update_vf_dir_credits(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_dqed_freelist_entries,
+					     dst->avail_dqed_freelist_entries,
+					     num);
+}
+
+int dlb_update_vf_hist_list_entries(struct dlb_hw *hw,
+				    u32 vf_id,
+				    u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_hist_list_entries,
+					     dst->avail_hist_list_entries,
+					     num);
+}
+
+int dlb_update_vf_atomic_inflights(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_aqed_freelist_entries,
+					     dst->avail_aqed_freelist_entries,
+					     num);
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (!queue) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (!queue)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id.phys_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id.phys_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id.phys_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id.phys_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id.phys_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (!port)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (!port)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (!pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (!pool)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (!pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (!pool)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id.phys_id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id.phys_id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id.phys_id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id.phys_id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id.phys_id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id.phys_id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id.phys_id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id.phys_id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id,
+						  vf_request,
+						  domain);
+
+		if (!port || port->domain_id.phys_id != domain->id.phys_id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args,
+				    bool vf_request,
+				    unsigned int vf_id)
+{
+	union dlb_sys_vf_ldb_vqid_v r0 = { {0} };
+	union dlb_sys_vf_ldb_vqid2qid r1 = { {0} };
+	union dlb_sys_ldb_qid2vqid r2 = { {0} };
+	union dlb_sys_ldb_vasqid_v r3 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r4 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r5 = { {0} };
+	union dlb_aqed_pipe_fl_lim r6 = { {0} };
+	union dlb_aqed_pipe_fl_base r7 = { {0} };
+	union dlb_chp_ord_qid_sn_map r11 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r12 = { {0} };
+	union dlb_sys_ldb_qid_v r13 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r14 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r15 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r16 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r17 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r3.field.vasqid_v = 0;
+
+	offs = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r3.val);
+
+	/* Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r4.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r4.val);
+
+	r5.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r5.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r5.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id.phys_id), r5.val);
+
+	r6.field.freelist_disable = 0;
+	r6.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id.phys_id), r6.val);
+
+	r7.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id.phys_id), r7.val);
+
+	r14.field.push_ptr = r7.field.base;
+	r14.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id.phys_id), r14.val);
+
+	r15.field.pop_ptr = r7.field.base;
+	r15.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id.phys_id), r15.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r11.field.mode = sn_group->mode;
+	r11.field.slot = queue->sn_slot;
+	r11.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id.phys_id), r11.val);
+
+	/* This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r16.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id.phys_id), r16.val);
+
+	r17.field.group = sn_group->id;
+	r17.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id.phys_id), r17.val);
+
+	r12.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r12.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id.phys_id), r12.val);
+
+	if (vf_request) {
+		unsigned int offs;
+
+		r0.field.vqid_v = 1;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VQID_V(offs), r0.val);
+
+		r1.field.qid = queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VQID2QID(offs), r1.val);
+
+		r2.field.vqid = queue->id.virt_id;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_QID2VQID(offs), r2.val);
+	}
+
+	r13.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id.phys_id), r13.val);
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue,
+				    bool vf_request,
+				    unsigned int vf_id)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS) + queue->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_dir_vqid_v   r1 = { {0} };
+		union dlb_sys_vf_dir_vqid2qid r2 = { {0} };
+
+		r1.field.vqid_v = 1;
+
+		offs = (vf_id * DLB_MAX_NUM_DIR_PORTS) + queue->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VQID_V(offs), r1.val);
+
+		r2.field.qid = queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VQID2QID(offs), r2.val);
+	} else {
+		union dlb_sys_dir_qid_v r3 = { {0} };
+
+		r3.field.qid_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id.phys_id), r3.val);
+	}
+
+	queue->queue_configured = true;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   vf_request,
+						   domain);
+
+		if (!queue || queue->domain_id.phys_id != domain->id.phys_id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       vf_request,
+				       domain);
+
+	if (!pool || !pool->configured ||
+	    pool->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp,
+				   bool vf_request,
+				   unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+
+	if (!queue || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id.phys_id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id.phys_id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id.phys_id, false, 0);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id.phys_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id.phys_id, port->id.phys_id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id.phys_id, port->id.phys_id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+
+	if (!queue || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+static int
+dlb_verify_enable_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_enable_ldb_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_enable_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_enable_dir_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_dir_pq_pair *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+
+	if (!port || !port->port_configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_disable_ldb_port_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_disable_ldb_port_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_disable_dir_port_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_disable_dir_port_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_dir_pq_pair *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+
+	if (!port || !port->port_configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_sys_ldb_pp2vpp r5 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+	unsigned int offs;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id.phys_id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id.phys_id), r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id.phys_id), r2.val);
+
+	r3.field.vas = domain->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id.phys_id), r3.val);
+
+	r5.field.vpp = port->id.virt_id;
+
+	offs = (vf_id * DLB_MAX_NUM_LDB_PORTS) + port->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VPP(offs), r5.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id.phys_id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id.phys_id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id.phys_id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id.phys_id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id.phys_id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id.phys_id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id.phys_id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id.phys_id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id.phys_id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id.phys_id), r18.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_ldb_vpp2pp r16 = { {0} };
+		union dlb_sys_vf_ldb_vpp_v r17 = { {0} };
+
+		r16.field.pp = port->id.phys_id;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP2PP(offs), r16.val);
+
+		r17.field.vpp_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP_V(offs), r17.val);
+	}
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id.phys_id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id.phys_id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id.phys_id),
+		   r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id.phys_id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id.phys_id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id.phys_id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id.phys_id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id.phys_id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id.phys_id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id.phys_id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id),
+		   r0.val);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id.phys_id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id.phys_id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw,
+					domain,
+					port,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_sys_dir_pp2vpp r5 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id.phys_id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id.phys_id),
+		   r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id.phys_id),
+		   r2.val);
+
+	r3.field.vas = domain->id.phys_id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id.phys_id),
+		   r3.val);
+
+	r5.field.vpp = port->id.virt_id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VPP((vf_id * DLB_MAX_NUM_DIR_PORTS) +
+				      port->id.phys_id),
+		   r5.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id.phys_id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id.phys_id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id.phys_id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id.phys_id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id.phys_id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id.phys_id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id.phys_id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id.phys_id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   r18.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_dir_vpp2pp r16 = { {0} };
+		union dlb_sys_vf_dir_vpp_v r17 = { {0} };
+		unsigned int offs;
+
+		r16.field.pp = port->id.phys_id;
+
+		offs = vf_id * DLB_MAX_NUM_DIR_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP2PP(offs), r16.val);
+
+		r17.field.vpp_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP_V(offs), r17.val);
+	}
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id.phys_id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id.phys_id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id.phys_id), r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id.phys_id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id.phys_id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id.phys_id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id.phys_id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					   vf_request,
+					   domain);
+	if (!dir_pool) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id.phys_id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args,
+					vf_request,
+					vf_id);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw,
+					domain,
+					port,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id.phys_id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id.phys_id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id.phys_id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id.phys_id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id.phys_id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id.phys_id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id.phys_id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id.phys_id,
+							   p->id.phys_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id.phys_id,
+						      p->id.phys_id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id.phys_id,
+						       p->id.phys_id / 4));
+
+	switch (p->id.phys_id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id.phys_id,
+						  p->id.phys_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id.phys_id,
+					     p->id.phys_id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id.phys_id,
+					      p->id.phys_id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id.phys_id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id.phys_id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id.phys_id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id.phys_id, false, 0);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id.phys_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id.phys_id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id.phys_id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id.phys_id;
+	queue_id = queue->id.phys_id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ * @vf_request: Request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = (vf_request) ? domain->id.virt_id : domain->id.phys_id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (!pool) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? pool->id.virt_id : pool->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (!pool) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? pool->id.virt_id : pool->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args, vf_request, vf_id);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? queue->id.virt_id : queue->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id,
+						   vf_request,
+						   domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue, vf_request, vf_id);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = (vf_request) ? queue->id.virt_id : queue->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args,
+					 bool vf_request,
+					 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? port->id.virt_id : port->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args,
+					 bool vf_request,
+					 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  vf_request,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = (vf_request) ? port->id.virt_id : port->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_start_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			__attribute((unused)) struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp,
+			bool vf_request,
+			unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_start_domain(hw, domain_id, vf_request, vf_id);
+
+	if (dlb_verify_start_domain_args(hw,
+					 domain_id,
+					 resp,
+					 vf_request,
+					 vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id.phys_id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id.phys_id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES +
+			ldb_queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS +
+			dir_queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it wasn't manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid, false, 0);
+
+		if (!queue) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n",
+		    args->priority);
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp,
+		   bool vf_request,
+		   unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw,
+				    domain_id,
+				    args,
+				    resp,
+				    vf_request,
+				    vf_id))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id.phys_id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp,
+		     bool vf_request,
+		     unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw,
+				      domain_id,
+				      args,
+				      resp,
+				      vf_request,
+				      vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: no available CQ slots\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asychronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_enable_port(struct dlb_hw *hw,
+				u32 domain_id,
+				u32 port_id,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB enable port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    port_id);
+}
+
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_enable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_enable_ldb_port_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (!port->enabled) {
+		dlb_ldb_port_cq_enable(hw, port);
+		port->enabled = true;
+
+		hw->pf.num_enabled_ldb_ports++;
+		dlb_update_ldb_arb_threshold(hw);
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_disable_port(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB disable port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    port_id);
+}
+
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_disable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_disable_ldb_port_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (port->enabled) {
+		dlb_ldb_port_cq_disable(hw, port);
+		port->enabled = false;
+
+		hw->pf.num_enabled_ldb_ports--;
+		dlb_update_ldb_arb_threshold(hw);
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_enable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_enable_dir_port_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (!port->enabled) {
+		dlb_dir_port_cq_enable(hw, port);
+		port->enabled = true;
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_disable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_disable_dir_port_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (port->enabled) {
+		dlb_dir_port_cq_disable(hw, port);
+		port->enabled = false;
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_notify_vf(struct dlb_hw *hw,
+		  unsigned int vf_id,
+		  enum dlb_mbox_vf_notification_type notification)
+{
+	struct dlb_mbox_vf_notification_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_NOTIFICATION;
+	req.notification = notification;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	/* No response data expected for notifications. */
+
+	return 0;
+}
+
+int dlb_vf_in_use(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_mbox_vf_in_use_cmd_resp resp;
+	struct dlb_mbox_vf_in_use_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_IN_USE;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	if (dlb_pf_read_vf_mbox_resp(hw, vf_id, &resp, sizeof(resp)))
+		return -1;
+
+	if (resp.hdr.status != DLB_MBOX_ST_SUCCESS) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: failed with mailbox error: %s\n",
+			   __func__,
+			   DLB_MBOX_ST_STRING(&resp));
+
+		return -1;
+	}
+
+	return resp.in_use;
+}
+
+static int dlb_vf_domain_alert(struct dlb_hw *hw,
+			       unsigned int vf_id,
+			       u32 domain_id,
+			       u32 alert_id,
+			       u32 aux_alert_data)
+{
+	struct dlb_mbox_vf_alert_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_DOMAIN_ALERT;
+	req.domain_id = domain_id;
+	req.alert_id = alert_id;
+	req.aux_alert_data = aux_alert_data;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	/* No response data expected for alarm notifications. */
+
+	return 0;
+}
+
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode)
+{
+	union dlb_sys_msix_mode r0 = { {0} };
+
+	r0.field.mode = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_MSIX_MODE, r0.val);
+}
+
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold)
+{
+	union dlb_chp_ldb_cq_int_depth_thrsh r0 = { {0} };
+	union dlb_chp_ldb_cq_int_enb r1 = { {0} };
+	union dlb_sys_ldb_cq_isr r2 = { {0} };
+	struct dlb_ldb_port *port;
+	bool vf_request;
+
+	vf_request = (mode == DLB_CQ_ISR_MODE_MSI);
+
+	port = dlb_get_ldb_port_from_id(hw, port_id, vf_request, vf);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: Internal error: failed to enable LDB CQ int\n\tport_id: %u, vf_req: %u, vf: %u\n",
+			   __func__, port_id, vf_request, vf);
+		return -EINVAL;
+	}
+
+	/* Trigger the interrupt when threshold or more QEs arrive in the CQ */
+	r0.field.depth_threshold = threshold - 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   r0.val);
+
+	r1.field.en_depth = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id), r1.val);
+
+	r2.field.vector = vector;
+	r2.field.vf = owner_vf;
+	r2.field.en_code = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_CQ_ISR(port->id.phys_id), r2.val);
+
+	return 0;
+}
+
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold)
+{
+	union dlb_chp_dir_cq_int_depth_thrsh r0 = { {0} };
+	union dlb_chp_dir_cq_int_enb r1 = { {0} };
+	union dlb_sys_dir_cq_isr r2 = { {0} };
+	struct dlb_dir_pq_pair *port;
+	bool vf_request;
+
+	vf_request = (mode == DLB_CQ_ISR_MODE_MSI);
+
+	port = dlb_get_dir_pq_from_id(hw, port_id, vf_request, vf);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: Internal error: failed to enable DIR CQ int\n\tport_id: %u, vf_req: %u, vf: %u\n",
+			   __func__, port_id, vf_request, vf);
+		return -EINVAL;
+	}
+
+	/* Trigger the interrupt when threshold or more QEs arrive in the CQ */
+	r0.field.depth_threshold = threshold - 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   r0.val);
+
+	r1.field.en_depth = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id), r1.val);
+
+	r2.field.vector = vector;
+	r2.field.vf = owner_vf;
+	r2.field.en_code = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ISR(port->id.phys_id), r2.val);
+
+	return 0;
+}
+
+int dlb_arm_cq_interrupt(struct dlb_hw *hw,
+			 int port_id,
+			 bool is_ldb,
+			 bool vf_request,
+			 unsigned int vf_id)
+{
+	u32 val;
+	u32 reg;
+
+	if (vf_request && is_ldb) {
+		struct dlb_ldb_port *ldb_port;
+
+		ldb_port = dlb_get_ldb_port_from_id(hw, port_id, true, vf_id);
+
+		if (!ldb_port || !ldb_port->configured)
+			return -EINVAL;
+
+		port_id = ldb_port->id.phys_id;
+	} else if (vf_request && !is_ldb) {
+		struct dlb_dir_pq_pair *dir_port;
+
+		dir_port = dlb_get_dir_pq_from_id(hw, port_id, true, vf_id);
+
+		if (!dir_port || !dir_port->port_configured)
+			return -EINVAL;
+
+		port_id = dir_port->id.phys_id;
+	}
+
+	val = 1 << (port_id % 32);
+
+	if (is_ldb && port_id < 32)
+		reg = DLB_CHP_LDB_CQ_INTR_ARMED0;
+	else if (is_ldb && port_id < 64)
+		reg = DLB_CHP_LDB_CQ_INTR_ARMED1;
+	else if (!is_ldb && port_id < 32)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED0;
+	else if (!is_ldb && port_id < 64)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED1;
+	else if (!is_ldb && port_id < 96)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED2;
+	else
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED3;
+
+	DLB_CSR_WR(hw, reg, val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts)
+{
+	/* Read every CQ's interrupt status */
+
+	ldb_interrupts[0] = DLB_CSR_RD(hw, DLB_SYS_LDB_CQ_31_0_OCC_INT_STS);
+	ldb_interrupts[1] = DLB_CSR_RD(hw, DLB_SYS_LDB_CQ_63_32_OCC_INT_STS);
+
+	dir_interrupts[0] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_31_0_OCC_INT_STS);
+	dir_interrupts[1] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_63_32_OCC_INT_STS);
+	dir_interrupts[2] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_95_64_OCC_INT_STS);
+	dir_interrupts[3] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_127_96_OCC_INT_STS);
+}
+
+static void dlb_ack_msix_interrupt(struct dlb_hw *hw, int vector)
+{
+	union dlb_sys_msix_ack r0 = { {0} };
+
+	switch (vector) {
+	case 0:
+		r0.field.msix_0_ack = 1;
+		break;
+	case 1:
+		r0.field.msix_1_ack = 1;
+		break;
+	case 2:
+		r0.field.msix_2_ack = 1;
+		break;
+	case 3:
+		r0.field.msix_3_ack = 1;
+		break;
+	case 4:
+		r0.field.msix_4_ack = 1;
+		break;
+	case 5:
+		r0.field.msix_5_ack = 1;
+		break;
+	case 6:
+		r0.field.msix_6_ack = 1;
+		break;
+	case 7:
+		r0.field.msix_7_ack = 1;
+		break;
+	case 8:
+		r0.field.msix_8_ack = 1;
+		/*
+		 * CSSY-1650
+		 * workaround h/w bug for lost MSI-X interrupts
+		 *
+		 * The recommended workaround for acknowledging
+		 * vector 8 interrupts is :
+		 *   1: set   MSI-X mask
+		 *   2: set   MSIX_PASSTHROUGH
+		 *   3: clear MSIX_ACK
+		 *   4: clear MSIX_PASSTHROUGH
+		 *   5: clear MSI-X mask
+		 *
+		 * The MSIX-ACK (step 3) is cleared for all vectors
+		 * below. We handle steps 1 & 2 for vector 8 here.
+		 *
+		 * The bitfields for MSIX_ACK and MSIX_PASSTHRU are
+		 * defined the same, so we just use the MSIX_ACK
+		 * value when writing to PASSTHRU.
+		 */
+
+		/* set MSI-X mask and passthrough for vector 8 */
+		DLB_FUNC_WR(hw, DLB_MSIX_MEM_VECTOR_CTRL(8), 1);
+		DLB_CSR_WR(hw, DLB_SYS_MSIX_PASSTHRU, r0.val);
+		break;
+	}
+
+	/* clear MSIX_ACK (write one to clear) */
+	DLB_CSR_WR(hw, DLB_SYS_MSIX_ACK, r0.val);
+
+	if (vector == 8) {
+		/*
+		 * finish up steps 4 & 5 of the workaround -
+		 * clear pasthrough and mask
+		 */
+		DLB_CSR_WR(hw, DLB_SYS_MSIX_PASSTHRU, 0);
+		DLB_FUNC_WR(hw, DLB_MSIX_MEM_VECTOR_CTRL(8), 0);
+	}
+
+	dlb_flush_csr(hw);
+}
+
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts)
+{
+	/* Write back the status regs to ack the interrupts */
+	if (ldb_interrupts[0])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_CQ_31_0_OCC_INT_STS,
+			   ldb_interrupts[0]);
+	if (ldb_interrupts[1])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_CQ_63_32_OCC_INT_STS,
+			   ldb_interrupts[1]);
+
+	if (dir_interrupts[0])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_31_0_OCC_INT_STS,
+			   dir_interrupts[0]);
+	if (dir_interrupts[1])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_63_32_OCC_INT_STS,
+			   dir_interrupts[1]);
+	if (dir_interrupts[2])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_95_64_OCC_INT_STS,
+			   dir_interrupts[2]);
+	if (dir_interrupts[3])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_127_96_OCC_INT_STS,
+			   dir_interrupts[3]);
+
+	dlb_ack_msix_interrupt(hw, DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID);
+}
+
+u32 dlb_read_vf_intr_status(struct dlb_hw *hw)
+{
+	return DLB_FUNC_RD(hw, DLB_FUNC_VF_VF_MSI_ISR);
+}
+
+void dlb_ack_vf_intr_status(struct dlb_hw *hw, u32 interrupts)
+{
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF_MSI_ISR, interrupts);
+}
+
+void dlb_ack_vf_msi_intr(struct dlb_hw *hw, u32 interrupts)
+{
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF_MSI_ISR_PEND, interrupts);
+}
+
+void dlb_ack_pf_mbox_int(struct dlb_hw *hw)
+{
+	union dlb_func_vf_pf2vf_mailbox_isr r0;
+
+	r0.field.pf_isr = 1;
+
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_PF2VF_MAILBOX_ISR, r0.val);
+}
+
+u32 dlb_read_vf_to_pf_int_bitvec(struct dlb_hw *hw)
+{
+	/* The PF has one VF->PF MBOX ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	return DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX_ISR(0));
+}
+
+void dlb_ack_vf_mbox_int(struct dlb_hw *hw, u32 bitvec)
+{
+	/* The PF has one VF->PF MBOX ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_MAILBOX_ISR(0), bitvec);
+}
+
+u32 dlb_read_vf_flr_int_bitvec(struct dlb_hw *hw)
+{
+	/* The PF has one VF->PF FLR ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	return DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_FLR_ISR(0));
+}
+
+void dlb_set_vf_reset_in_progress(struct dlb_hw *hw, int vf)
+{
+	u32 bitvec = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+
+	bitvec |= (1 << vf);
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), bitvec);
+}
+
+void dlb_clr_vf_reset_in_progress(struct dlb_hw *hw, int vf)
+{
+	u32 bitvec = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+
+	bitvec &= ~(1 << vf);
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), bitvec);
+}
+
+void dlb_ack_vf_flr_int(struct dlb_hw *hw, u32 bitvec, bool a_stepping)
+{
+	union dlb_sys_func_vf_bar_dsbl r0 = { {0} };
+	u32 clear;
+	int i;
+
+	if (!bitvec)
+		return;
+
+	/* Re-enable access to the VF BAR */
+	r0.field.func_vf_bar_dis = 0;
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		if (!(bitvec & (1 << i)))
+			continue;
+
+		DLB_CSR_WR(hw, DLB_SYS_FUNC_VF_BAR_DSBL(i), r0.val);
+	}
+
+	/* Notify the VF driver that the reset has completed. This register is
+	 * RW in A-stepping devices, WOCLR otherwise.
+	 */
+	if (a_stepping) {
+		clear = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+		clear &= ~bitvec;
+	} else {
+		clear = bitvec;
+	}
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), clear);
+
+	/* Mark the FLR ISR as complete */
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_FLR_ISR(0), bitvec);
+}
+
+void dlb_ack_vf_to_pf_int(struct dlb_hw *hw,
+			  u32 mbox_bitvec,
+			  u32 flr_bitvec)
+{
+	int i;
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_VF_TO_PF_MBOX);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		union dlb_func_pf_vf2pf_isr_pend r0 = { {0} };
+
+		if (!((mbox_bitvec & (1 << i)) || (flr_bitvec & (1 << i))))
+			continue;
+
+		/* Unset the VF's ISR pending bit */
+		r0.field.isr_pend = 1;
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_ISR_PEND(i), r0.val);
+	}
+}
+
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw)
+{
+	union dlb_sys_ingress_alarm_enbl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_INGRESS_ALARM_ENBL);
+
+	r0.field.illegal_hcw = 1;
+	r0.field.illegal_pp = 1;
+	r0.field.disabled_pp = 1;
+	r0.field.illegal_qid = 1;
+	r0.field.disabled_qid = 1;
+	r0.field.illegal_ldb_qid_cfg = 1;
+	r0.field.illegal_cqid = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_INGRESS_ALARM_ENBL, r0.val);
+}
+
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw)
+{
+	union dlb_sys_ingress_alarm_enbl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_INGRESS_ALARM_ENBL);
+
+	r0.field.illegal_hcw = 0;
+	r0.field.illegal_pp = 0;
+	r0.field.disabled_pp = 0;
+	r0.field.illegal_qid = 0;
+	r0.field.disabled_qid = 0;
+	r0.field.illegal_ldb_qid_cfg = 0;
+	r0.field.illegal_cqid = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_INGRESS_ALARM_ENBL, r0.val);
+}
+
+static void dlb_log_alarm_syndrome(struct dlb_hw *hw,
+				   const char *str,
+				   union dlb_sys_alarm_hw_synd r0)
+{
+	DLB_HW_ERR(hw, "%s:\n", str);
+	DLB_HW_ERR(hw, "\tsyndrome: 0x%x\n", r0.field.syndrome);
+	DLB_HW_ERR(hw, "\trtype:    0x%x\n", r0.field.rtype);
+	DLB_HW_ERR(hw, "\tfrom_dmv: 0x%x\n", r0.field.from_dmv);
+	DLB_HW_ERR(hw, "\tis_ldb:   0x%x\n", r0.field.is_ldb);
+	DLB_HW_ERR(hw, "\tcls:      0x%x\n", r0.field.cls);
+	DLB_HW_ERR(hw, "\taid:      0x%x\n", r0.field.aid);
+	DLB_HW_ERR(hw, "\tunit:     0x%x\n", r0.field.unit);
+	DLB_HW_ERR(hw, "\tsource:   0x%x\n", r0.field.source);
+	DLB_HW_ERR(hw, "\tmore:     0x%x\n", r0.field.more);
+	DLB_HW_ERR(hw, "\tvalid:    0x%x\n", r0.field.valid);
+}
+
+/* Note: this array's contents must match dlb_alert_id() */
+static const char dlb_alert_strings[NUM_DLB_DOMAIN_ALERTS][128] = {
+	[DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS] = "Insufficient credits",
+	[DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ] = "Illegal enqueue",
+	[DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS] = "Excess token pops",
+	[DLB_DOMAIN_ALERT_ILLEGAL_HCW] = "Illegal HCW",
+	[DLB_DOMAIN_ALERT_ILLEGAL_QID] = "Illegal QID",
+	[DLB_DOMAIN_ALERT_DISABLED_QID] = "Disabled QID",
+};
+
+static void dlb_log_pf_vf_syndrome(struct dlb_hw *hw,
+				   const char *str,
+				   union dlb_sys_alarm_pf_synd0 r0,
+				   union dlb_sys_alarm_pf_synd1 r1,
+				   union dlb_sys_alarm_pf_synd2 r2,
+				   u32 alert_id)
+{
+	DLB_HW_ERR(hw, "%s:\n", str);
+	if (alert_id < NUM_DLB_DOMAIN_ALERTS)
+		DLB_HW_ERR(hw, "Alert: %s\n", dlb_alert_strings[alert_id]);
+	DLB_HW_ERR(hw, "\tsyndrome:     0x%x\n", r0.field.syndrome);
+	DLB_HW_ERR(hw, "\trtype:        0x%x\n", r0.field.rtype);
+	DLB_HW_ERR(hw, "\tfrom_dmv:     0x%x\n", r0.field.from_dmv);
+	DLB_HW_ERR(hw, "\tis_ldb:       0x%x\n", r0.field.is_ldb);
+	DLB_HW_ERR(hw, "\tcls:          0x%x\n", r0.field.cls);
+	DLB_HW_ERR(hw, "\taid:          0x%x\n", r0.field.aid);
+	DLB_HW_ERR(hw, "\tunit:         0x%x\n", r0.field.unit);
+	DLB_HW_ERR(hw, "\tsource:       0x%x\n", r0.field.source);
+	DLB_HW_ERR(hw, "\tmore:         0x%x\n", r0.field.more);
+	DLB_HW_ERR(hw, "\tvalid:        0x%x\n", r0.field.valid);
+	DLB_HW_ERR(hw, "\tdsi:          0x%x\n", r1.field.dsi);
+	DLB_HW_ERR(hw, "\tqid:          0x%x\n", r1.field.qid);
+	DLB_HW_ERR(hw, "\tqtype:        0x%x\n", r1.field.qtype);
+	DLB_HW_ERR(hw, "\tqpri:         0x%x\n", r1.field.qpri);
+	DLB_HW_ERR(hw, "\tmsg_type:     0x%x\n", r1.field.msg_type);
+	DLB_HW_ERR(hw, "\tlock_id:      0x%x\n", r2.field.lock_id);
+	DLB_HW_ERR(hw, "\tmeas:         0x%x\n", r2.field.meas);
+	DLB_HW_ERR(hw, "\tdebug:        0x%x\n", r2.field.debug);
+	DLB_HW_ERR(hw, "\tcq_pop:       0x%x\n", r2.field.cq_pop);
+	DLB_HW_ERR(hw, "\tqe_uhl:       0x%x\n", r2.field.qe_uhl);
+	DLB_HW_ERR(hw, "\tqe_orsp:      0x%x\n", r2.field.qe_orsp);
+	DLB_HW_ERR(hw, "\tqe_valid:     0x%x\n", r2.field.qe_valid);
+	DLB_HW_ERR(hw, "\tcq_int_rearm: 0x%x\n", r2.field.cq_int_rearm);
+	DLB_HW_ERR(hw, "\tdsi_error:    0x%x\n", r2.field.dsi_error);
+}
+
+static void dlb_clear_syndrome_register(struct dlb_hw *hw, u32 offset)
+{
+	union dlb_sys_alarm_hw_synd r0 = { {0} };
+
+	r0.field.valid = 1;
+	r0.field.more = 1;
+
+	DLB_CSR_WR(hw, offset, r0.val);
+}
+
+void dlb_process_alarm_interrupt(struct dlb_hw *hw)
+{
+	union dlb_sys_alarm_hw_synd r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_HW_SYND);
+
+	dlb_log_alarm_syndrome(hw, "HW alarm syndrome", r0);
+
+	dlb_clear_syndrome_register(hw, DLB_SYS_ALARM_HW_SYND);
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_ALARM);
+}
+
+static void dlb_process_ingress_error(struct dlb_hw *hw,
+				      union dlb_sys_alarm_pf_synd0 r0,
+				      u32 alert_id,
+				      bool vf_error,
+				      unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	bool is_ldb;
+	u8 port_id;
+	int ret;
+
+	port_id = r0.field.syndrome & 0x7F;
+	if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS)
+		is_ldb = r0.field.is_ldb;
+	else
+		is_ldb = (r0.field.syndrome & 0x80) != 0;
+
+	/* Get the domain ID and, if it's a VF domain, the virtual port ID */
+	if (is_ldb) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_ldb_port_from_id(hw, port_id, vf_error, vf_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()]: Internal error: unable to find LDB port\n\tport: %u, vf_error: %u, vf_id: %u\n",
+				   __func__, port_id, vf_error, vf_id);
+			return;
+		}
+
+		domain = &hw->domains[port->domain_id.phys_id];
+	} else {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_dir_pq_from_id(hw, port_id, vf_error, vf_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()]: Internal error: unable to find DIR port\n\tport: %u, vf_error: %u, vf_id: %u\n",
+				   __func__, port_id, vf_error, vf_id);
+			return;
+		}
+
+		domain = &hw->domains[port->domain_id.phys_id];
+	}
+
+	if (vf_error)
+		ret = dlb_vf_domain_alert(hw,
+					  vf_id,
+					  domain->id.virt_id,
+					  alert_id,
+					  (is_ldb << 8) | port_id);
+	else
+		ret = os_notify_user_space(hw,
+					   domain->id.phys_id,
+					   alert_id,
+					   (is_ldb << 8) | port_id);
+
+	if (ret)
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to notify\n",
+			   __func__);
+}
+
+static u32 dlb_alert_id(union dlb_sys_alarm_pf_synd0 r0)
+{
+	if (r0.field.unit == DLB_ALARM_HW_UNIT_CHP &&
+	    r0.field.aid == DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS)
+		return DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS;
+	else if (r0.field.unit == DLB_ALARM_HW_UNIT_CHP &&
+		 r0.field.aid == DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ)
+		return DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ;
+	else if (r0.field.unit == DLB_ALARM_HW_UNIT_LSP &&
+		 r0.field.aid == DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS)
+		return DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_ILLEGAL_HCW)
+		return DLB_DOMAIN_ALERT_ILLEGAL_HCW;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_ILLEGAL_QID)
+		return DLB_DOMAIN_ALERT_ILLEGAL_QID;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_DISABLED_QID)
+		return DLB_DOMAIN_ALERT_DISABLED_QID;
+	else
+		return NUM_DLB_DOMAIN_ALERTS;
+}
+
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw)
+{
+	union dlb_sys_alarm_pf_synd0 r0;
+	union dlb_sys_alarm_pf_synd1 r1;
+	union dlb_sys_alarm_pf_synd2 r2;
+	u32 alert_id;
+	int i;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND0);
+
+	if (r0.field.valid) {
+		r1.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND1);
+		r2.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND2);
+
+		alert_id = dlb_alert_id(r0);
+
+		dlb_log_pf_vf_syndrome(hw,
+				       "PF Ingress error alarm",
+				       r0, r1, r2, alert_id);
+
+		dlb_clear_syndrome_register(hw, DLB_SYS_ALARM_PF_SYND0);
+
+		dlb_process_ingress_error(hw, r0, alert_id, false, 0);
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND0(i));
+
+		if (!r0.field.valid)
+			continue;
+
+		r1.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND1(i));
+		r2.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND2(i));
+
+		alert_id = dlb_alert_id(r0);
+
+		dlb_log_pf_vf_syndrome(hw,
+				       "VF Ingress error alarm",
+				       r0, r1, r2, alert_id);
+
+		dlb_clear_syndrome_register(hw,
+					    DLB_SYS_ALARM_VF_SYND0(i));
+
+		dlb_process_ingress_error(hw, r0, alert_id, true, i);
+	}
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_INGRESS_ERROR);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val &= ~(1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT);
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id.phys_id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id.phys_id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue __attribute__((unused));
+	struct dlb_dir_pq_pair *tmp_dir_port __attribute__((unused));
+	struct dlb_ldb_port *tmp_ldb_port __attribute__((unused));
+	struct dlb_credit_pool *tmp_pool __attribute__((unused));
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+void dlb_resource_reset(struct dlb_hw *hw)
+{
+	struct dlb_domain *domain, *next __attribute__((unused));
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		DLB_FUNC_LIST_FOR_SAFE(hw->vf[i].used_domains, domain,
+				       next, iter1, iter2)
+			dlb_domain_reset_software_state(hw, domain);
+	}
+
+	DLB_FUNC_LIST_FOR_SAFE(hw->pf.used_domains, domain, next, iter1, iter2)
+		dlb_domain_reset_software_state(hw, domain);
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id.phys_id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id,
+				    vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, id, vf_request, vf_id);
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!queue) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args, vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, vf_request, domain);
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id.phys_id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id.phys_id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_req,
+			       unsigned int vf_id)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id,
+				    vf_req, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_req, vf_id);
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, vf_req, domain);
+	if (!queue) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id.phys_id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id.phys_id));
+
+	return r0.field.count;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id.phys_id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id.phys_id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id.phys_id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id.phys_id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id.phys_id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id.phys_id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id.phys_id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id.phys_id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id.phys_id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id.phys_id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id.phys_id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id.phys_id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id.phys_id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id.phys_id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id.phys_id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id.phys_id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id.phys_id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id.phys_id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id.phys_id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id.phys_id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id.phys_id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id.phys_id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id.phys_id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id.phys_id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id.phys_id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id.phys_id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id.phys_id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id.phys_id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id.phys_id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id.phys_id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id.phys_id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id.phys_id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id.phys_id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id.phys_id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id.phys_id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id.phys_id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id.phys_id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id.phys_id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id.phys_id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id.phys_id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_vpps(struct dlb_hw *hw,
+					struct dlb_domain *domain,
+					unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_vf_dir_vpp_v r1;
+	struct dlb_dir_pq_pair *port;
+
+	r1.field.vpp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		unsigned int offs;
+
+		offs = vf_id * DLB_MAX_NUM_DIR_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP_V(offs), r1.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_vpps(struct dlb_hw *hw,
+					struct dlb_domain *domain,
+					unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_vf_ldb_vpp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.vpp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		unsigned int offs;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP_V(offs), r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id.phys_id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id.phys_id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id.phys_id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw,
+		     u32 domain_id,
+		     bool vf_request,
+		     unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id, vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	if (vf_request) {
+		dlb_domain_disable_dir_vpps(hw, domain, vf_id);
+
+		dlb_domain_disable_ldb_vpps(hw, domain, vf_id);
+	}
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process doesn't decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during VAS reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int dlb_reset_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_domain *domain, *next __attribute__((unused));
+	struct dlb_list_entry *it1 __attribute__((unused));
+	struct dlb_list_entry *it2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+
+	if (vf_id >= DLB_MAX_NUM_VFS) {
+		DLB_HW_ERR(hw, "[%s()] Internal error: invalid VF ID %d\n",
+			   __func__, vf_id);
+		return -EFAULT;
+	}
+
+	rsrcs = &hw->vf[vf_id];
+
+	DLB_FUNC_LIST_FOR_SAFE(rsrcs->used_domains, domain, next, it1, it2) {
+		int ret = dlb_reset_domain(hw,
+					   domain->id.virt_id,
+					   true,
+					   vf_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	port = dlb_get_domain_ldb_port(port_id, vf_request, domain);
+
+	if (!port)
+		return -EINVAL;
+
+	return port->domain_id.phys_id == domain->id.phys_id;
+}
+
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	port = dlb_get_domain_dir_pq(port_id, vf_request, domain);
+
+	if (!port)
+		return -EINVAL;
+
+	return port->domain_id.phys_id == domain->id.phys_id;
+}
+
+int dlb_hw_get_num_resources(struct dlb_hw *hw,
+			     struct dlb_get_num_resources_args *arg,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	if (vf_request)
+		rsrcs = &hw->vf[vf_id];
+	else
+		rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+
+	return 0;
+}
+
+int dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				  struct dlb_get_num_resources_args *arg,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	memset(arg, 0, sizeof(*arg));
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		struct dlb_dir_pq_pair *dir_port;
+		struct dlb_ldb_port *ldb_port;
+		struct dlb_credit_pool *pool;
+		struct dlb_ldb_queue *queue;
+
+		arg->num_sched_domains++;
+
+		arg->num_atomic_inflights +=
+			domain->aqed_freelist.bound -
+			domain->aqed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
+			arg->num_ldb_queues++;
+		DLB_DOM_LIST_FOR(domain->avail_ldb_queues, queue, iter2)
+			arg->num_ldb_queues++;
+
+		DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter2)
+			arg->num_ldb_ports++;
+		DLB_DOM_LIST_FOR(domain->avail_ldb_ports, ldb_port, iter2)
+			arg->num_ldb_ports++;
+
+		DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter2)
+			arg->num_dir_ports++;
+		DLB_DOM_LIST_FOR(domain->avail_dir_pq_pairs, dir_port, iter2)
+			arg->num_dir_ports++;
+
+		arg->num_ldb_credits +=
+			domain->qed_freelist.bound -
+			domain->qed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->avail_ldb_credit_pools, pool, iter2)
+			arg->num_ldb_credit_pools++;
+		DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter2) {
+			arg->num_ldb_credit_pools++;
+			arg->num_ldb_credits += pool->total_credits;
+		}
+
+		arg->num_dir_credits +=
+			domain->dqed_freelist.bound -
+			domain->dqed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->avail_dir_credit_pools, pool, iter2)
+			arg->num_dir_credit_pools++;
+		DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter2) {
+			arg->num_dir_credit_pools++;
+			arg->num_dir_credits += pool->total_credits;
+		}
+
+		arg->num_hist_list_entries += domain->total_hist_list_entries;
+	}
+
+	return 0;
+}
+
+static inline bool dlb_ldb_port_owned_by_vf(struct dlb_hw *hw,
+					    u32 vf_id,
+					    u32 port_id)
+{
+	return (hw->rsrcs.ldb_ports[port_id].id.vf_owned &&
+		hw->rsrcs.ldb_ports[port_id].id.vf_id == vf_id);
+}
+
+static inline bool dlb_dir_port_owned_by_vf(struct dlb_hw *hw,
+					    u32 vf_id,
+					    u32 port_id)
+{
+	return (hw->rsrcs.dir_pq_pairs[port_id].id.vf_owned &&
+		hw->rsrcs.dir_pq_pairs[port_id].id.vf_id == vf_id);
+}
+
+void dlb_send_async_pf_to_vf_msg(struct dlb_hw *hw, unsigned int vf_id)
+{
+	union dlb_func_pf_pf2vf_mailbox_isr r0 = { {0} };
+
+	r0.field.isr = 1 << vf_id;
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX_ISR(0), r0.val);
+}
+
+bool dlb_pf_to_vf_complete(struct dlb_hw *hw, unsigned int vf_id)
+{
+	union dlb_func_pf_pf2vf_mailbox_isr r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_PF_PF2VF_MAILBOX_ISR(vf_id));
+
+	return (r0.val & (1 << vf_id)) == 0;
+}
+
+void dlb_send_async_vf_to_pf_msg(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf2pf_mailbox_isr r0 = { {0} };
+
+	r0.field.isr = 1;
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX_ISR, r0.val);
+}
+
+bool dlb_vf_to_pf_complete(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf2pf_mailbox_isr r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_VF_VF2PF_MAILBOX_ISR);
+
+	return (r0.field.isr == 0);
+}
+
+bool dlb_vf_flr_complete(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf_reset_in_progress r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_VF_VF_RESET_IN_PROGRESS);
+
+	return (r0.field.reset_in_progress == 0);
+}
+
+int dlb_pf_read_vf_mbox_req(struct dlb_hw *hw,
+			    unsigned int vf_id,
+			    void *data,
+			    int len)
+{
+	u32 buf[DLB_VF2PF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	if (len == 0) {
+		DLB_HW_ERR(hw, "[%s()] invalid len (0)\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_REQ_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_pf_read_vf_mbox_resp(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len)
+{
+	u32 buf[DLB_VF2PF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_RESP_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_pf_write_vf_mbox_resp(struct dlb_hw *hw,
+			      unsigned int vf_id,
+			      void *data,
+			      int len)
+{
+	u32 buf[DLB_PF2VF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_RESP_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_pf_write_vf_mbox_req(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len)
+{
+	u32 buf[DLB_PF2VF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_REQ_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_vf_read_pf_mbox_resp(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_PF2VF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	if (len == 0) {
+		DLB_HW_ERR(hw, "[%s()] invalid len (0)\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_RESP_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_VF_PF2VF_MAILBOX(idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_vf_read_pf_mbox_req(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_PF2VF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if ((len % 4) != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_REQ_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_VF_PF2VF_MAILBOX(idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_vf_write_pf_mbox_req(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_VF2PF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_REQ_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX(idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_vf_write_pf_mbox_resp(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_VF2PF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_RESP_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX(idx), buf[i]);
+	}
+
+	return 0;
+}
+
+bool dlb_vf_is_locked(struct dlb_hw *hw, unsigned int vf_id)
+{
+	return hw->vf[vf_id].locked;
+}
+
+static void dlb_vf_set_rsrc_virt_ids(struct dlb_function_resources *rsrcs,
+				     unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	int i;
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_domains, domain, iter) {
+		domain->id.virt_id = i;
+		domain->id.vf_owned = true;
+		domain->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, ldb_queue, iter) {
+		ldb_queue->id.virt_id = i;
+		ldb_queue->id.vf_owned = true;
+		ldb_queue->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, ldb_port, iter) {
+		ldb_port->id.virt_id = i;
+		ldb_port->id.vf_owned = true;
+		ldb_port->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_pq_pairs, dir_port, iter) {
+		dir_port->id.virt_id = i;
+		dir_port->id.vf_owned = true;
+		dir_port->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_credit_pools, pool, iter) {
+		pool->id.virt_id = i;
+		pool->id.vf_owned = true;
+		pool->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_credit_pools, pool, iter) {
+		pool->id.virt_id = i;
+		pool->id.vf_owned = true;
+		pool->id.vf_id = vf_id;
+		i++;
+	}
+}
+
+void dlb_lock_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_function_resources *rsrcs = &hw->vf[vf_id];
+
+	rsrcs->locked = true;
+
+	dlb_vf_set_rsrc_virt_ids(rsrcs, vf_id);
+}
+
+void dlb_unlock_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	hw->vf[vf_id].locked = false;
+}
+
+int dlb_reset_vf_resources(struct dlb_hw *hw, unsigned int vf_id)
+{
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	dlb_update_vf_sched_domains(hw, vf_id, 0);
+	dlb_update_vf_ldb_queues(hw, vf_id, 0);
+	dlb_update_vf_ldb_ports(hw, vf_id, 0);
+	dlb_update_vf_dir_ports(hw, vf_id, 0);
+	dlb_update_vf_ldb_credit_pools(hw, vf_id, 0);
+	dlb_update_vf_dir_credit_pools(hw, vf_id, 0);
+	dlb_update_vf_ldb_credits(hw, vf_id, 0);
+	dlb_update_vf_dir_credits(hw, vf_id, 0);
+	dlb_update_vf_hist_list_entries(hw, vf_id, 0);
+	dlb_update_vf_atomic_inflights(hw, vf_id, 0);
+
+	return 0;
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8])
+{
+	union dlb_atm_pipe_ctrl_arb_weights_rdy_bin r0 = { {0} };
+	union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 r1 = { {0} };
+	union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 r2 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 r3 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 r4 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 r5 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 r6 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 r7 =  { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 r8 =  { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 r9 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 r10 = { {0} };
+	union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin r11 = { {0} };
+	union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 r12 = { {0} };
+
+	r0.field.bin0 = weight[1];
+	r0.field.bin1 = weight[3];
+	r0.field.bin2 = weight[5];
+	r0.field.bin3 = weight[7];
+
+	r1.field.pri0 = weight[0];
+	r1.field.pri1 = weight[1];
+	r1.field.pri2 = weight[2];
+	r1.field.pri3 = weight[3];
+	r2.field.pri4 = weight[4];
+	r2.field.pri5 = weight[5];
+	r2.field.pri6 = weight[6];
+	r2.field.pri7 = weight[7];
+
+	r3.field.pri0 = weight[0];
+	r3.field.pri1 = weight[1];
+	r3.field.pri2 = weight[2];
+	r3.field.pri3 = weight[3];
+	r4.field.pri4 = weight[4];
+	r4.field.pri5 = weight[5];
+	r4.field.pri6 = weight[6];
+	r4.field.pri7 = weight[7];
+
+	r5.field.pri0 = weight[0];
+	r5.field.pri1 = weight[1];
+	r5.field.pri2 = weight[2];
+	r5.field.pri3 = weight[3];
+	r6.field.pri4 = weight[4];
+	r6.field.pri5 = weight[5];
+	r6.field.pri6 = weight[6];
+	r6.field.pri7 = weight[7];
+
+	r7.field.pri0 = weight[0];
+	r7.field.pri1 = weight[1];
+	r7.field.pri2 = weight[2];
+	r7.field.pri3 = weight[3];
+	r8.field.pri4 = weight[4];
+	r8.field.pri5 = weight[5];
+	r8.field.pri6 = weight[6];
+	r8.field.pri7 = weight[7];
+
+	r9.field.pri0 = weight[0];
+	r9.field.pri1 = weight[1];
+	r9.field.pri2 = weight[2];
+	r9.field.pri3 = weight[3];
+	r10.field.pri4 = weight[4];
+	r10.field.pri5 = weight[5];
+	r10.field.pri6 = weight[6];
+	r10.field.pri7 = weight[7];
+
+	r11.field.bin0 = weight[1];
+	r11.field.bin1 = weight[3];
+	r11.field.bin2 = weight[5];
+	r11.field.bin3 = weight[7];
+
+	r12.field.pri0 = weight[1];
+	r12.field.pri1 = weight[3];
+	r12.field.pri2 = weight[5];
+	r12.field.pri3 = weight[7];
+
+	DLB_CSR_WR(hw, DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN, r0.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0, r1.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1, r2.val);
+	DLB_CSR_WR(hw,
+		   DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0,
+		   r3.val);
+	DLB_CSR_WR(hw,
+		   DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1,
+		   r4.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0, r5.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1, r6.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0, r7.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1, r8.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0, r9.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1, r10.val);
+	DLB_CSR_WR(hw, DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN, r11.val);
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0, r12.val);
+}
+
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8])
+{
+	union dlb_lsp_cfg_arb_weight_ldb_qid_0 r0 = { {0} };
+	union dlb_lsp_cfg_arb_weight_ldb_qid_1 r1 = { {0} };
+	union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 r2 = { {0} };
+	union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 r3 = { {0} };
+
+	r0.field.slot0_weight = weight[0];
+	r0.field.slot1_weight = weight[1];
+	r0.field.slot2_weight = weight[2];
+	r0.field.slot3_weight = weight[3];
+	r1.field.slot4_weight = weight[4];
+	r1.field.slot5_weight = weight[5];
+	r1.field.slot6_weight = weight[6];
+	r1.field.slot7_weight = weight[7];
+
+	r2.field.slot0_weight = weight[0];
+	r2.field.slot1_weight = weight[1];
+	r2.field.slot2_weight = weight[2];
+	r2.field.slot3_weight = weight[3];
+	r3.field.slot4_weight = weight[4];
+	r3.field.slot5_weight = weight[5];
+	r3.field.slot6_weight = weight[6];
+	r3.field.slot7_weight = weight[7];
+
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0, r0.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1, r1.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0, r2.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1, r3.val);
+}
+
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_ldb_pp_sw_alarm_en r0 = { {0} };
+	union dlb_chp_cfg_dir_pp_sw_alarm_en r1 = { {0} };
+	int i;
+
+	r0.field.alarm_enable = 1;
+	r1.field.alarm_enable = 1;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(i), r0.val);
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(i), r1.val);
+}
+
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_ldb_pp_sw_alarm_en r0 = { {0} };
+	union dlb_chp_cfg_dir_pp_sw_alarm_en r1 = { {0} };
+	int i;
+
+	r0.field.alarm_enable = 0;
+	r1.field.alarm_enable = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(i), r0.val);
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(i), r1.val);
+}
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 000000000..5500f9b26
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,1625 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+#include "dlb_user.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ * All resources go back to their owning function, whether a PF or a VF.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp,
+			bool vf_request,
+			unsigned int vf_id);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp,
+		   bool vf_request,
+		   unsigned int vf_id);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp,
+		     bool vf_request,
+		     unsigned int vf_id);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @vf: If the port is VF-owned, the VF's ID. This is used for translating the
+ *	virtual port ID to a physical port ID. Ignored if mode is not MSI.
+ * @owner_vf: the VF to route the interrupt to. Ignore if mode is not MSI.
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @vf: If the port is VF-owned, the VF's ID. This is used for translating the
+ *	virtual port ID to a physical port ID. Ignored if mode is not MSI.
+ * @owner_vf: the VF to route the interrupt to. Ignore if mode is not MSI.
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw,
+			 int port_id,
+			 bool is_ldb,
+			 bool vf_request,
+			 unsigned int vf_id);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_read_vf_intr_status() - read the VF interrupt status register
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function can be called from a VF's interrupt handler to determine
+ * which interrupts have fired. The first 31 bits correspond to CQ interrupt
+ * vectors, and the final bit is for the PF->VF mailbox interrupt vector.
+ *
+ * Return:
+ * Returns a bit vector indicating which interrupt vectors are active.
+ */
+u32 dlb_read_vf_intr_status(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_intr_status() - ack VF interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @interrupts: 32-bit bitmap
+ *
+ * This function ACKs a VF's interrupts. Its interrupts argument should be the
+ * value returned by dlb_read_vf_intr_status().
+ */
+void dlb_ack_vf_intr_status(struct dlb_hw *hw, u32 interrupts);
+
+/**
+ * dlb_ack_vf_msi_intr() - ack VF MSI interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @interrupts: 32-bit bitmap
+ *
+ * This function clears the VF's MSI interrupt pending register. Its interrupts
+ * argument should be contain the MSI vectors to ACK. For example, if MSI MME
+ * is in mode 0, then one bit 0 should ever be set.
+ */
+void dlb_ack_vf_msi_intr(struct dlb_hw *hw, u32 interrupts);
+
+/**
+ * dlb_ack_vf_mbox_int() - ack PF->VF mailbox interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When done processing the PF mailbox request, this function unsets
+ * the PF's mailbox ISR register.
+ */
+void dlb_ack_pf_mbox_int(struct dlb_hw *hw);
+
+/**
+ * dlb_read_vf_to_pf_int_bitvec() - return a bit vector of all requesting VFs
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When the VF->PF ISR fires, this function can be called to determine which
+ * VF(s) are requesting service. This bitvector must be passed to
+ * dlb_ack_vf_to_pf_int() when processing is complete for all requesting VFs.
+ *
+ * Return:
+ * Returns a bit vector indicating which VFs (0-15) have requested service.
+ */
+u32 dlb_read_vf_to_pf_int_bitvec(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_mbox_int() - ack processed VF->PF mailbox interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @bitvec: bit vector returned by dlb_read_vf_to_pf_int_bitvec()
+ *
+ * When done processing all VF mailbox requests, this function unsets the VF's
+ * mailbox ISR register.
+ */
+void dlb_ack_vf_mbox_int(struct dlb_hw *hw, u32 bitvec);
+
+/**
+ * dlb_read_vf_flr_int_bitvec() - return a bit vector of all VFs requesting FLR
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When the VF FLR ISR fires, this function can be called to determine which
+ * VF(s) are requesting FLRs. This bitvector must passed to
+ * dlb_ack_vf_flr_int() when processing is complete for all requesting VFs.
+ *
+ * Return:
+ * Returns a bit vector indicating which VFs (0-15) have requested FLRs.
+ */
+u32 dlb_read_vf_flr_int_bitvec(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_flr_int() - ack processed VF<->PF interrupt(s)
+ * @hw: dlb_hw handle for a particular device.
+ * @bitvec: bit vector returned by dlb_read_vf_flr_int_bitvec()
+ * @a_stepping: device is A-stepping
+ *
+ * When done processing all VF FLR requests, this function unsets the VF's FLR
+ * ISR register.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_ack_vf_flr_int(struct dlb_hw *hw, u32 bitvec, bool a_stepping);
+
+/**
+ * dlb_ack_vf_to_pf_int() - ack processed VF mbox and FLR interrupt(s)
+ * @hw: dlb_hw handle for a particular device.
+ * @mbox_bitvec: bit vector returned by dlb_read_vf_to_pf_int_bitvec()
+ * @flr_bitvec: bit vector returned by dlb_read_vf_flr_int_bitvec()
+ *
+ * When done processing all VF requests, this function communicates to the
+ * hardware that processing is complete. When this function completes, hardware
+ * can immediately generate another VF mbox or FLR interrupt.
+ */
+void dlb_ack_vf_to_pf_int(struct dlb_hw *hw,
+			  u32 mbox_bitvec,
+			  u32 flr_bitvec);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw,
+		     u32 domain_id,
+		     bool vf_request,
+		     unsigned int vf_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: indicates whether this request came from a VF.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: indicates whether this request came from a VF.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the number of available resources for the PF or for a
+ * VF.
+ *
+ * Return:
+ * Returns 0 upon success, -1 if vf_request is true and vf_id is invalid.
+ */
+int dlb_hw_get_num_resources(struct dlb_hw *hw,
+			     struct dlb_get_num_resources_args *arg,
+			     bool vf_request,
+			     unsigned int vf_id);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the number of resources in use by the PF or a VF. It
+ * fills in the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ *
+ * Return:
+ * Returns 0 upon success, -1 if vf_request is true and vf_id is invalid.
+ */
+int dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				  struct dlb_get_num_resources_args *arg,
+				  bool vf_request,
+				  unsigned int vf_id);
+
+/**
+ * dlb_send_async_vf_to_pf_msg() - (VF only) send a mailbox message to the PF
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function sends a VF->PF mailbox message. It is asynchronous, so it
+ * returns once the message is sent but potentially before the PF has processed
+ * the message. The caller must call dlb_vf_to_pf_complete() to determine when
+ * the PF has finished processing the request.
+ */
+void dlb_send_async_vf_to_pf_msg(struct dlb_hw *hw);
+
+/**
+ * dlb_vf_to_pf_complete() - check the status of an asynchronous mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether the PF has finished
+ * processing a VF->PF mailbox request. It should only be called after sending
+ * an asynchronous request with dlb_send_async_vf_to_pf_msg().
+ */
+bool dlb_vf_to_pf_complete(struct dlb_hw *hw);
+
+/**
+ * dlb_vf_flr_complete() - check the status of a VF FLR
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether the PF has finished
+ * executing the VF FLR. It should only be called after setting the VF's FLR
+ * bit.
+ */
+bool dlb_vf_flr_complete(struct dlb_hw *hw);
+
+/**
+ * dlb_set_vf_reset_in_progress() - set a VF's reset in progress bit
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * Note: This function is only supported on A-stepping devices.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_set_vf_reset_in_progress(struct dlb_hw *hw, int vf_id);
+
+/**
+ * dlb_clr_vf_reset_in_progress() - clear a VF's reset in progress bit
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * Note: This function is only supported on A-stepping devices.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_clr_vf_reset_in_progress(struct dlb_hw *hw, int vf_id);
+
+/**
+ * dlb_send_async_pf_to_vf_msg() - (PF only) send a mailbox message to the VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * This function sends a PF->VF mailbox message. It is asynchronous, so it
+ * returns once the message is sent but potentially before the VF has processed
+ * the message. The caller must call dlb_pf_to_vf_complete() to determine when
+ * the VF has finished processing the request.
+ */
+void dlb_send_async_pf_to_vf_msg(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_pf_to_vf_complete() - check the status of an asynchronous mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * This function returns a boolean indicating whether the VF has finished
+ * processing a PF->VF mailbox request. It should only be called after sending
+ * an asynchronous request with dlb_send_async_pf_to_vf_msg().
+ */
+bool dlb_pf_to_vf_complete(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_pf_read_vf_mbox_req() - (PF only) read a VF->PF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies one of the PF's VF->PF mailboxes into the array pointed
+ * to by data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_REQ_BYTES.
+ */
+int dlb_pf_read_vf_mbox_req(struct dlb_hw *hw,
+			    unsigned int vf_id,
+			    void *data,
+			    int len);
+
+/**
+ * dlb_pf_read_vf_mbox_resp() - (PF only) read a VF->PF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies one of the PF's VF->PF mailboxes into the array pointed
+ * to by data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_RESP_BYTES.
+ */
+int dlb_pf_read_vf_mbox_resp(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len);
+
+/**
+ * dlb_pf_write_vf_mbox_resp() - (PF only) write a PF->VF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the PF's VF->PF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_RESP_BYTES.
+ */
+int dlb_pf_write_vf_mbox_resp(struct dlb_hw *hw,
+			      unsigned int vf_id,
+			      void *data,
+			      int len);
+
+/**
+ * dlb_pf_write_vf_mbox_req() - (PF only) write a PF->VF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the PF's VF->PF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_REQ_BYTES.
+ */
+int dlb_pf_write_vf_mbox_req(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len);
+
+/**
+ * dlb_vf_read_pf_mbox_resp() - (VF only) read a PF->VF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the VF's PF->VF mailbox into the array pointed to by
+ * data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_RESP_BYTES.
+ */
+int dlb_vf_read_pf_mbox_resp(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_read_pf_mbox_req() - (VF only) read a PF->VF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the VF's PF->VF mailbox into the array pointed to by
+ * data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_REQ_BYTES.
+ */
+int dlb_vf_read_pf_mbox_req(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_write_pf_mbox_req() - (VF only) write a VF->PF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the VF's PF->VF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_REQ_BYTES.
+ */
+int dlb_vf_write_pf_mbox_req(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_write_pf_mbox_resp() - (VF only) write a VF->PF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the VF's PF->VF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_RESP_BYTES.
+ */
+int dlb_vf_write_pf_mbox_resp(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_reset_vf() - reset the hardware owned by a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function resets the hardware owned by a VF (if any), by resetting the
+ * VF's domains one by one.
+ */
+int dlb_reset_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_vf_is_locked() - check whether the VF's resources are locked
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function returns whether or not the VF's resource assignments are
+ * locked. If locked, no resources can be added to or subtracted from the
+ * group.
+ */
+bool dlb_vf_is_locked(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_lock_vf() - lock the VF's resources
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function sets a flag indicating that the VF is using its resources.
+ * When VF is locked, its resource assignment cannot be changed.
+ */
+void dlb_lock_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_unlock_vf() - unlock the VF's resources
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function unlocks the VF's resource assignment, allowing it to be
+ * modified.
+ */
+void dlb_unlock_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_update_vf_sched_domains() - update the domains assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of scheduling domains to assign to this VF
+ *
+ * This function assigns num scheduling domains to the specified VF. If the VF
+ * already has domains assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_sched_domains(struct dlb_hw *hw,
+				u32 vf_id,
+				u32 num);
+
+/**
+ * dlb_update_vf_ldb_queues() - update the LDB queues assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB queues to assign to this VF
+ *
+ * This function assigns num LDB queues to the specified VF. If the VF already
+ * has LDB queues assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_queues(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_ldb_ports() - update the LDB ports assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB ports to assign to this VF
+ *
+ * This function assigns num LDB ports to the specified VF. If the VF already
+ * has LDB ports assigned, this existing assignment is adjusted accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_ports(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_dir_ports() - update the DIR ports assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR ports to assign to this VF
+ *
+ * This function assigns num DIR ports to the specified VF. If the VF already
+ * has DIR ports assigned, this existing assignment is adjusted accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_ports(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_ldb_credit_pools() - update the VF's assigned LDB pools
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB credit pools to assign to this VF
+ *
+ * This function assigns num LDB credit pools to the specified VF. If the VF
+ * already has LDB credit pools assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_update_vf_dir_credit_pools() - update the VF's assigned DIR pools
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR credit pools to assign to this VF
+ *
+ * This function assigns num DIR credit pools to the specified VF. If the VF
+ * already has DIR credit pools assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_update_vf_ldb_credits() - update the VF's assigned LDB credits
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB credits to assign to this VF
+ *
+ * This function assigns num LDB credits to the specified VF. If the VF already
+ * has LDB credits assigned, this existing assignment is adjusted accordingly.
+ * VF's are assigned a contiguous chunk of credits, so this function may fail
+ * if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_credits(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_dir_credits() - update the VF's assigned DIR credits
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR credits to assign to this VF
+ *
+ * This function assigns num DIR credits to the specified VF. If the VF already
+ * has DIR credits assigned, this existing assignment is adjusted accordingly.
+ * VF's are assigned a contiguous chunk of credits, so this function may fail
+ * if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_credits(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_hist_list_entries() - update the VF's assigned HL entries
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of history list entries to assign to this VF
+ *
+ * This function assigns num history list entries to the specified VF. If the
+ * VF already has history list entries assigned, this existing assignment is
+ * adjusted accordingly. VF's are assigned a contiguous chunk of entries, so
+ * this function may fail if a sufficiently large contiguous chunk is not
+ * available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_hist_list_entries(struct dlb_hw *hw,
+				    u32 vf_id,
+				    u32 num);
+
+/**
+ * dlb_update_vf_atomic_inflights() - update the VF's atomic inflights
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of atomic inflights to assign to this VF
+ *
+ * This function assigns num atomic inflights to the specified VF. If the VF
+ * already has atomic inflights assigned, this existing assignment is adjusted
+ * accordingly. VF's are assigned a contiguous chunk of entries, so this
+ * function may fail if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_atomic_inflights(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_reset_vf_resources() - reassign the VF's resources to the PF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function takes any resources currently assigned to the VF and reassigns
+ * them to the PF.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_reset_vf_resources(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_notify_vf() - send a notification to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @notification: notification
+ *
+ * This function sends a notification (as defined in dlb_mbox.h) to a VF.
+ *
+ * Return:
+ * Returns 0 upon success, -1 if the VF doesn't ACK the PF->VF interrupt.
+ */
+int dlb_notify_vf(struct dlb_hw *hw,
+		  unsigned int vf_id,
+		  u32 notification);
+
+/**
+ * dlb_vf_in_use() - query whether a VF is in use
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function sends a mailbox request to the VF to query whether the VF is in
+ * use.
+ *
+ * Return:
+ * Returns 0 for false, 1 for true, and -1 if the mailbox request times out or
+ * an internal error occurs.
+ */
+int dlb_vf_in_use(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_user.h b/drivers/event/dlb/pf/base/dlb_user.h
new file mode 100644
index 000000000..6e7ee2ec3
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_user.h
@@ -0,0 +1,1084 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#define DLB_MAX_NAME_LEN 64
+
+#include "dlb_osdep_types.h"
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' device file commands */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_GET_DEVICE_VERSION: Query the DLB device version.
+ *
+ *	This ioctl interface is the same in all driver versions and is always
+ *	the first ioctl.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id[7:0]: Device revision.
+ *	response.id[15:8]: Device version.
+ */
+
+struct dlb_get_device_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+#define DLB_VERSION_MAJOR_NUMBER 10
+#define DLB_VERSION_MINOR_NUMBER 7
+#define DLB_VERSION_REVISION_NUMBER 9
+#define DLB_VERSION (DLB_VERSION_MAJOR_NUMBER << 24 | \
+		     DLB_VERSION_MINOR_NUMBER << 16 | \
+		     DLB_VERSION_REVISION_NUMBER)
+
+#define DLB_VERSION_GET_MAJOR_NUMBER(x) (((x) >> 24) & 0xFF)
+#define DLB_VERSION_GET_MINOR_NUMBER(x) (((x) >> 16) & 0xFF)
+#define DLB_VERSION_GET_REVISION_NUMBER(x) ((x) & 0xFFFF)
+
+static inline __u8 dlb_version_incompatible(__u32 version)
+{
+	__u8 inc;
+
+	inc = DLB_VERSION_GET_MAJOR_NUMBER(version) != DLB_VERSION_MAJOR_NUMBER;
+	inc |= (int)DLB_VERSION_GET_MINOR_NUMBER(version) <
+		DLB_VERSION_MINOR_NUMBER;
+
+	return inc;
+}
+
+/*
+ * DLB_CMD_GET_DRIVER_VERSION: Query the DLB driver version. The major number
+ *	is changed when there is an ABI-breaking change, the minor number is
+ *	changed if the API is changed in a backwards-compatible way, and the
+ *	revision number is changed for fixes that don't affect the API.
+ *
+ *	If the kernel driver's API version major number and the header's
+ *	DLB_VERSION_MAJOR_NUMBER differ, the two are incompatible, or if the
+ *	major numbers match but the kernel driver's minor number is less than
+ *	the header file's, they are incompatible. The DLB_VERSION_INCOMPATIBLE
+ *	macro should be used to check for compatibility.
+ *
+ *	This ioctl interface is the same in all driver versions. Applications
+ *	should check the driver version before performing any other ioctl
+ *	operations.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Driver API version. Use the DLB_VERSION_GET_MAJOR_NUMBER,
+ *		DLB_VERSION_GET_MINOR_NUMBER, and
+ *		DLB_VERSION_GET_REVISION_NUMBER macros to interpret the field.
+ */
+
+struct dlb_get_driver_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_user_interface_commands {
+	DLB_CMD_GET_DEVICE_VERSION,
+	DLB_CMD_CREATE_SCHED_DOMAIN,
+	DLB_CMD_GET_NUM_RESOURCES,
+	DLB_CMD_GET_DRIVER_VERSION,
+	DLB_CMD_SAMPLE_PERF_COUNTERS,
+	DLB_CMD_SET_SN_ALLOCATION,
+	DLB_CMD_GET_SN_ALLOCATION,
+	DLB_CMD_MEASURE_SCHED_COUNTS,
+	DLB_CMD_QUERY_CQ_POLL_MODE,
+	DLB_CMD_GET_SN_OCCUPANCY,
+
+	/* NUM_DLB_CMD must be last */
+	NUM_DLB_CMD,
+};
+
+/*******************************/
+/* 'domain' device file alerts */
+/*******************************/
+
+/* Scheduling domain device files can be read to receive domain-specific
+ * notifications, for alerts such as hardware errors.
+ *
+ * Each alert is encoded in a 16B message. The first 8B contains the alert ID,
+ * and the second 8B is optional and contains additional information.
+ * Applications should cast read data to a struct dlb_domain_alert, and
+ * interpret the struct's alert_id according to dlb_domain_alert_id. The read
+ * length must be 16B, or the function will return -EINVAL.
+ *
+ * Reads are destructive, and in the case of multiple file descriptors for the
+ * same domain device file, an alert will be read by only one of the file
+ * descriptors.
+ *
+ * The driver stores alerts in a fixed-size alert ring until they are read. If
+ * the alert ring fills completely, subsequent alerts will be dropped. It is
+ * recommended that DLB applications dedicate a thread to perform blocking
+ * reads on the device file.
+ */
+enum dlb_domain_alert_id {
+	/* A destination domain queue that this domain connected to has
+	 * unregistered, and can no longer be sent to. The aux alert data
+	 * contains the queue ID.
+	 */
+	DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER,
+	/* A producer port in this domain attempted to send a QE without a
+	 * credit. aux_alert_data[7:0] contains the port ID, and
+	 * aux_alert_data[15:8] contains a flag indicating whether the port is
+	 * load-balanced (1) or directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS,
+	/* Software issued an illegal enqueue for a port in this domain. An
+	 * illegal enqueue could be:
+	 * - Illegal (excess) completion
+	 * - Illegal fragment
+	 * - Illegal enqueue command
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ,
+	/* Software issued excess CQ token pops for a port in this domain.
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS,
+	/* A enqueue contained either an invalid command encoding or a REL,
+	 * REL_T, RLS, FWD, FWD_T, FRAG, or FRAG_T from a directed port.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_HCW,
+	/* The QID must be valid and less than 128.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_QID,
+	/* An enqueue went to a disabled QID.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_DISABLED_QID,
+	/* The device containing this domain was reset. All applications using
+	 * the device need to exit for the driver to complete the reset
+	 * procedure.
+	 *
+	 * aux_alert_data doesn't contain any information for this alert.
+	 */
+	DLB_DOMAIN_ALERT_DEVICE_RESET,
+	/* User-space has enqueued an alert.
+	 *
+	 * aux_alert_data contains user-provided data.
+	 */
+	DLB_DOMAIN_ALERT_USER,
+
+	/* Number of DLB domain alerts */
+	NUM_DLB_DOMAIN_ALERTS
+};
+
+static const char dlb_domain_alert_strings[][128] = {
+	"DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER",
+	"DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS",
+	"DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ",
+	"DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS",
+	"DLB_DOMAIN_ALERT_ILLEGAL_HCW",
+	"DLB_DOMAIN_ALERT_ILLEGAL_QID",
+	"DLB_DOMAIN_ALERT_DISABLED_QID",
+	"DLB_DOMAIN_ALERT_DEVICE_RESET",
+	"DLB_DOMAIN_ALERT_USER",
+};
+
+struct dlb_domain_alert {
+	__u64 alert_id;
+	__u64 aux_alert_data;
+};
+
+/*********************************/
+/* 'domain' device file commands */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT: Block on a CQ interrupt until a QE
+ *	arrives for the specified port. If a QE is already present, the ioctl
+ *	will immediately return.
+ *
+ *	Note: Only one thread can block on a CQ's interrupt at a time. Doing
+ *	otherwise can result in hung threads.
+ *
+ * Input parameters:
+ * - port_id: Port ID.
+ * - is_ldb: True if the port is load-balanced, false otherwise.
+ * - arm: Tell the driver to arm the interrupt.
+ * - cq_gen: Current CQ generation bit.
+ * - padding0: Reserved for future use.
+ * - cq_va: VA of the CQ entry where the next QE will be placed.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_block_on_cq_interrupt_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u8 is_ldb;
+	__u8 arm;
+	__u8 cq_gen;
+	__u8 padding0;
+	__u64 cq_va;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT: Enqueue a domain alert that will be
+ *	read by one reader thread.
+ *
+ * Input parameters:
+ * - aux_alert_data: user-defined auxiliary data.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enqueue_domain_alert_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u64 aux_alert_data;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+enum dlb_domain_user_interface_commands {
+	DLB_DOMAIN_CMD_CREATE_LDB_POOL,
+	DLB_DOMAIN_CMD_CREATE_DIR_POOL,
+	DLB_DOMAIN_CMD_CREATE_LDB_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_DIR_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_LDB_PORT,
+	DLB_DOMAIN_CMD_CREATE_DIR_PORT,
+	DLB_DOMAIN_CMD_START_DOMAIN,
+	DLB_DOMAIN_CMD_MAP_QID,
+	DLB_DOMAIN_CMD_UNMAP_QID,
+	DLB_DOMAIN_CMD_ENABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_ENABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_DISABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_DISABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT,
+	DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT,
+	DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS,
+
+	/* NUM_DLB_DOMAIN_CMD must be last */
+	NUM_DLB_DOMAIN_CMD,
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
-- 
2.13.6

[dpdk-dev] [PATCH 04/27] event/dlb: add make and meson build infrastructure

[McDaniel, Timothy]

Change-Id: I7fc0656b531dded6a95fbb0777c3b0dd5209ba61
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 config/common_base            | 17 +++++++++++++++++
 config/rte_config.h           |  6 ++++++
 drivers/event/Makefile        |  5 +++++
 drivers/event/dlb/Makefile    | 36 ++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build | 16 ++++++++++++++++
 drivers/event/meson.build     |  3 +++
 6 files changed, 83 insertions(+)
 create mode 100644 drivers/event/dlb/Makefile
 create mode 100644 drivers/event/dlb/meson.build

diff --git a/config/common_base b/config/common_base
index c7d5c7321..cb2d0dd0c 100644
--- a/config/common_base
+++ b/config/common_base
@@ -789,6 +789,23 @@ CONFIG_RTE_LIBRTE_PMD_DPAA_EVENTDEV=n
 CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV=n
 
 #
+# Compile PMD for dlb event device
+#
+CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV=y
+# Specify an upper bound on cycles between each CQ access when hard polling
+CONFIG_RTE_LIBRTE_PMD_DLB_POLL_INTERVAL=1000
+# Specify control state to use when executing UMWAIT instruction.
+# If 0,  then use CO.2, which has a slower wakeup time, but greater
+# power savings.
+# If 1, then use C0.1, which has a faster wakeup time, but smaller
+# power savings.
+CONFIG_RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE=0
+# Suppress statistics calculation
+CONFIG_RTE_LIBRTE_PMD_DLB_QUELL_STATS=n
+# Specify software credit quanta
+CONFIG_RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA=32
+
+#
 # Compile raw device support
 # EXPERIMENTAL: API may change without prior notice
 #
diff --git a/config/rte_config.h b/config/rte_config.h
index e9201fd46..839c0001d 100644
--- a/config/rte_config.h
+++ b/config/rte_config.h
@@ -133,4 +133,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
diff --git a/drivers/event/Makefile b/drivers/event/Makefile
index 86be41b9e..125b5e8fa 100644
--- a/drivers/event/Makefile
+++ b/drivers/event/Makefile
@@ -16,5 +16,10 @@ ifeq ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_EVENTDEV) += dpaa2
 endif
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OPDL_EVENTDEV) += opdl
+# DLB limited to x86 architectures
+ifneq ($(filter y,$(CONFIG_RTE_ARCH_X86_64) \
+		  $(CONFIG_RTE_ARCH_X86)),)
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb
+endif
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/event/dlb/Makefile b/drivers/event/dlb/Makefile
new file mode 100644
index 000000000..f32235a46
--- /dev/null
+++ b/drivers/event/dlb/Makefile
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2016-2020 Intel Corporation.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_dlb_eventdev.a
+
+# build flags
+CFLAGS += -O3 -pthread
+CFLAGS += $(WERROR_FLAGS)
+# rte_mp_ APIs are still marked experimental
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+LDLIBS += -lrte_eal -lrte_eventdev -lrte_mbuf -lrte_kvargs -lrte_ring
+LDLIBS += -lrte_mempool -lpthread -lrte_pci -lrte_bus_pci
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_pmd_dlb_event_version.map
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb_iface.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb_xstats.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += rte_pmd_dlb.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb_selftest.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += pf/dlb_pf.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += pf/dlb_main.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += pf/base/dlb_resource.c
+
+# export include files
+SYMLINK-y-include += rte_pmd_dlb.h
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
new file mode 100644
index 000000000..9c6a5d6a1
--- /dev/null
+++ b/drivers/event/dlb/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+sources = files('dlb.c',
+		'dlb_iface.c',
+		'dlb_xstats.c',
+		'rte_pmd_dlb.c',
+		'dlb_selftest.c',
+		'pf/dlb_pf.c',
+		'pf/dlb_main.c',
+		'pf/base/dlb_resource.c'
+)
+
+allow_experimental_apis = true
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
diff --git a/drivers/event/meson.build b/drivers/event/meson.build
index 50d30c53f..decad5569 100644
--- a/drivers/event/meson.build
+++ b/drivers/event/meson.build
@@ -6,6 +6,9 @@ if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
 endif
+if dpdk_conf.has('RTE_ARCH_X86_64') or dpdk_conf.has('RTE_ARCH_X86')
+	drivers += 'dlb'
+endif
 std_deps = ['eventdev', 'kvargs']
 config_flag_fmt = 'RTE_LIBRTE_@0@_EVENTDEV_PMD'
 driver_name_fmt = 'rte_pmd_@0@_event'
-- 
2.13.6

[dpdk-dev] [PATCH 05/27] event/dlb: add DLB documentation

[McDaniel, Timothy]

Change-Id: I269bfe9fb4ac53a9f81d33718f3a808fd8216c74
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 doc/guides/eventdevs/dlb.rst | 497 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 497 insertions(+)
 create mode 100644 doc/guides/eventdevs/dlb.rst

diff --git a/doc/guides/eventdevs/dlb.rst b/doc/guides/eventdevs/dlb.rst
new file mode 100644
index 000000000..21e48fea0
--- /dev/null
+++ b/doc/guides/eventdevs/dlb.rst
@@ -0,0 +1,497 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+.. note::
+
+    This PMD is disabled by default in the build configuration files, owing to
+    an external dependency on the `Netlink Protocol Library Suite
+    <http://www.infradead.org/~tgr/libnl/>`_ (libnl-3 and libnl-genl-3) which
+    must be installed on the board.  Once the Netlink libraries are installed,
+    the PMD can be enabled by setting CONFIG_RTE_LIBRTE_PMD_DLB_QM=y and
+    recompiling the DPDK.
+
+Prerequisites
+-------------
+
+- Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+  the basic DPDK environment.
+
+- Learn about the DLB device and its capabilities at `Intel Support
+  <http://www.intel.com/support>`_. FIXME: Add real link when documentation
+  becomes available.
+
+- The DLB kernel module. If it is not included in the machine's OS
+  distribution, download it from <FIXME: Add 01.org link when available> and
+  follow the build instructions.
+
+Configuration
+-------------
+
+The DLB eventdev supports two modes of operation:
+
+* Bifurcated mode: the PMD is created as a vdev device and depends on the Linux
+  DLB kernel driver for device access. The bifurcated PMD's configuration
+  accesses are performed through the kernel driver, and (performance-critical)
+  datapath functions execute entirely in user-space.
+
+  This mode supports both PF and VF devices, but is supported on Linux only.
+
+* PF PMD mode: the PF PMD is a user-space PMD that uses VFIO to gain direct
+  device access. To use this operation mode, the PCIe PF device must be bound
+  to a DPDK-compatible VFIO driver, such as vfio-pci. The PF PMD does not work
+  with PCIe VFs, but is portable to all environments (Linux, FreeBSD, etc.)
+  that DPDK supports. (Note: PF PMD testing has been limited to Linux at this
+  time.)
+
+The vdev device can be created from the application code or from the EAL
+command line like so:
+
+* Call ``rte_vdev_init("dlb1_event")`` from the application.
+
+* Use ``--vdev="dlb1_event"`` in the EAL options, which will call
+  rte_vdev_init() internally.
+
+Example:
+
+.. code-block:: console
+
+    ./your_eventdev_application --vdev="dlb1_event"
+
+Note: The dlb vdev can be instatiated with the name "event_dlb" as well.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and doesn't require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
+Wait (timeout_ticks) Parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The eventdev API rte_event_dequeue_burst(..) can wait for an event to
+arrive. Three different forms of waiting are supported by the dlb PMD:
+polling, blocking on a hardware interrupt, and waiting using umonitor/umwait.
+Which form of wait to use can be specified using the hybrid timeout data
+structure below.The application should use the appropriate bybrid timeout
+struct below and cast it to uint32_t or uint64_t,  as appropriate.
+
+If RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT is set, then the timeout_ticks
+parameter supplied to rte_event_dequeue_burst(..) is used to control if and how
+to wait, and dequeue_timeout_ns is ignored.
+
+If RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT is not set, then dequeue_timeout_ns
+supplied to the rte_event_dev_configure API is used to control if and how to
+wait, and the timeout_ticks value is ignored.
+
+The application should use the appropriate bybrid timeout struct below and cast
+it to uint32_t (for rte_event_dev_configure) or uint64_t (for
+rte_event_dequeue_burst),  as appropriate.
+
+Hybrid timeout_ticks
+^^^^^^^^^^^^^^^^^^^^
+#. If poll_ticks is not 0 and neither interrupt_wait or umonitor_wait are set,
+   then we will busy poll for up to poll_ticks time.
+#. If the interrupt_wait bit is set and the CQ is empty, then enter kernel
+   to wait for an interrupt after busy polling for poll_ticks time. There
+   is no guarantee how much time we spend in the API when using interrupt_wait.
+#. If umonitor_wait is set, then repeatedly issue a umwait instruction
+   until the requested number of events have been dequeued,  or until
+   poll_ticks has expired.
+
+Note: It is invalid to set both interrupt_wait and umonitor_wait.
+
+The hybrid timeout data structures are currently located in
+drivers/event/dlb/dlb_timeout.h:
+
+.. code-block:: c
+
+        struct rte_hybrid_timeout_ticks_64 {
+                RTE_STD_C11
+                union {
+                        uint64_t val64;
+                        struct {
+                                uint64_t poll_ticks:62;
+                                uint64_t umonitor_wait:1;
+                                uint64_t interrupt_wait:1;
+                        };
+                };
+        };
+        struct rte_hybrid_timeout_ns_32 {
+                RTE_STD_C11
+                union {
+                        uint32_t val32;
+                        struct {
+                                uint32_t poll_ns:30;
+                                uint32_t umonitor_wait:1;
+                                uint32_t interrupt_wait:1;
+                        };
+                };
+        };
+
+VAS Configuration
+~~~~~~~~~~~~~~~~~
+
+A VAS is a scheduling domain, of which there are 32 in the DLB. (Producer
+ports in one VAS cannot enqueue events to a different VAS, except through the
+`Data Mover`_.) When a VAS is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some VAS
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+VAS), are not.
+
+The dlb PMD creates a single VAS per DLB device. Supporting multiple VASes
+per DLB device is a planned feature, where each VAS will be represented as a
+separate event device.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a VAS is created in ``rte_event_dev_configure()``, the user specifies
+``nb_event_ports`` and ``nb_single_link_event_port_queues``, which control the
+total number of ports (load-balanced and directed) and the number of directed
+ports. Hence, the number of requested load-balanced ports is ``nb_event_ports
+- nb_single_link_event_ports``. The ``nb_event_queues`` field specifies the
+total number of queues (load-balanced and directed). The number of directed
+queues comes from ``nb_single_link_event_port_queues``, since directed ports
+and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API doesn't have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+An dlb eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overriden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 8 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB doesn't limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB doesn't support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
+Ordered Fragments
+~~~~~~~~~~~~~~~~~
+
+The DLB has a fourth enqueue type: partial enqueue. When a thread is processing
+an ordered event, it can perform up to 16 "partial" enqueues, which allows a
+single received ordered event to result in multiple reordered events.
+
+For example, consider the case where three events (A, then B, then C) are
+enqueued with ordered scheduling and are received by three different ports.
+The ports that receive A and C forward events A' and C', while the port that
+receives B generates three partial enqueues -- B1', B2', and B3' -- followed by
+a release operation. The DLB will reorder the events in the following order:
+
+A', B1', B2', B3', C'
+
+This functionality is not available explicitly through the eventdev API, but
+the dlb PMD provides it through an additional (DLB-specific) event operation,
+RTE_EVENT_DLB_OP_FRAG.
+
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
-- 
2.13.6

[dpdk-dev] [PATCH 06/27] event/dlb: add dynamic logging

[McDaniel, Timothy]

Change-Id: I7698d9b8eea2e681ea6ff237d1f93b0ece7f4704
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_log.h | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_log.h

diff --git a/drivers/event/dlb/dlb_log.h b/drivers/event/dlb/dlb_log.h
new file mode 100644
index 000000000..b8f4ba5cf
--- /dev/null
+++ b/drivers/event/dlb/dlb_log.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+                __func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_DBG(fmt, args...) \
+	DLB_LOG_IMPL(DEBUG, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
-- 
2.13.6

[dpdk-dev] [PATCH 07/27] event/dlb: add private data structures and constants

[McDaniel, Timothy]

Change-Id: I6be9078848e25c19ce3b357491f7f616e1cf675d
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_priv.h | 595 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 595 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h

diff --git a/drivers/event/dlb/dlb_priv.h b/drivers/event/dlb/dlb_priv.h
new file mode 100644
index 000000000..f9c3cdc04
--- /dev/null
+++ b/drivers/event/dlb/dlb_priv.h
@@ -0,0 +1,595 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <rte_eventdev.h>
+#include <stdbool.h>
+#include "rte_config.h"
+#include "dlb_user.h"
+#include "dlb_log.h"
+#include "rte_pmd_dlb.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/*  command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NOTIFY_ENTRY_OFFSET 2048
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/*
+ * Static per queue/port provisioning values
+ */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define CQ_BASE(is_dir) ((is_dir) ? DLB_DIR_CQ_BASE : DLB_LDB_CQ_BASE)
+#define CQ_SIZE(is_dir) ((is_dir) ? DLB_DIR_CQ_MAX_SIZE : DLB_LDB_CQ_MAX_SIZE)
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+#define PC_BASE(is_dir) ((is_dir) ? DLB_DIR_PC_BASE : DLB_LDB_PC_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+#define DLB_1K 1024
+#define DLB_2K (2 * DLB_1K)
+#define DLB_4K (4 * DLB_1K)
+#define DLB_16K (16 * DLB_1K)
+#define DLB_32K (32 * DLB_1K)
+#define DLB_1MB (DLB_1K * DLB_1K)
+#define DLB_16MB (16 * DLB_1MB)
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+/* vdev args */
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/**> Total queues (lb + dir) */
+	uint32_t num_ldb_queues;	/**> Number of available ldb queues */
+	uint32_t num_ldb_ports;         /**< Number of load balanced ports */
+	uint32_t num_dir_ports;         /**< Number of directed ports */
+	uint32_t num_ldb_credits;       /**< Number of load balanced credits */
+	uint32_t num_dir_credits;       /**< Number of directed credits */
+	uint32_t reorder_window_size;   /**< Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this QM instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+struct dlb_hw_queue_params {
+	 /**> qid type: atomic, ordered, etc. */
+	enum dlb_hw_sched_type type;
+	/**> size of the reorder window */
+	uint32_t reorder_window_size;
+	/**> priority of this QID:
+	 *  0   = highest
+	 *  128 = default
+	 *  255 = lowest
+	 */
+	uint8_t priority;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+union dlb_port_config {
+	struct dlb_create_ldb_port_args ldb;
+	struct dlb_create_dir_port_args dir;
+};
+
+enum DLB_PORT_STATE {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
+	union dlb_port_config cfg;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int16_t token_pop_thresh;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum DLB_PORT_STATE state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	bool mmaped;
+};
+
+struct dlb_eventdev;
+
+struct dlb_port_low_level_io_functions {
+	void (*pp_enqueue_four)(struct dlb_enqueue_qe *qe4, uint64_t *pp_addr);
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	char device_name[DLB_NAME_SIZE];
+	char device_path[DLB_MAX_DEVICE_PATH];
+	int device_path_id;
+	char domain_device_path[DLB_MAX_DEVICE_PATH];
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+}; __rte_cache_aligned
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/* Used for parsing dir ports/queues. */
+
+/* Note: eventdev currently is limited to 64 ports, but DLB hardware has 128
+ * directed ports/queues.
+ */
+struct dlb_dir_resource_list {
+	int entries;
+	uint32_t id[DLB_MAX_NUM_DIR_PORTS];
+};
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+struct dlb_device_stats {
+	/* Device specific */
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+#define DLB_IS_VDEV true
+#define DLB_NOT_VDEV false
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	rte_atomic32_t inflights;
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+	/* Device stats */
+	struct dlb_device_stats stats __rte_cache_aligned;
+};
+
+/* End Eventdev related defines and structs */
+
+/* Forwards for non-inlined functions */
+
+void dlb_free_qe_mem(struct dlb_port *port);
+
+int dlb_init_qe_mem(struct dlb_port *port, char *mz_name);
+
+int dlb_init_send_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_init_partial_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_init_fwd_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_init_consume_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_init_complete_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_init_noop_qe(struct dlb_port *port, char *mz_name);
+
+int dlb_uninit(const char *name);
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstat_names,
+		unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+		const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode,
+		int16_t queue_port_id,
+		const uint32_t ids[],
+		uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+				const char *name,
+				struct dlb_devargs *dlb_args,
+				bool is_vdev);
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				  const char *name,
+				  bool is_vdev);
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		     struct dlb_eventdev_queue *queue);
+
+/* arg parsing helper functions */
+int set_numa_node(const char *key __rte_unused, const char *value,
+		     void *opaque);
+int set_dir_ports(const char *key __rte_unused,
+		  const char *value __rte_unused,
+		  void *opaque __rte_unused);
+int set_dir_queues(const char *key __rte_unused,
+		   const char *value __rte_unused,
+		   void *opaque __rte_unused);
+int set_max_num_events(const char *key __rte_unused, const char *value,
+		       void *opaque);
+int set_num_dir_credits(const char *key __rte_unused, const char *value,
+			void *opaque);
+void dlb_drain(struct rte_eventdev *dev);
+void dlb_entry_points_init(struct rte_eventdev *dev);
+int dlb_parse_params(const char *params,
+	       const char *name,
+	       struct dlb_devargs *dlb_args);
+int dlb_string_to_int(int *result, const char *str);
+
+#endif	/* _DLB_PRIV_H_ */
-- 
2.13.6

[dpdk-dev] [PATCH 08/27] event/dlb: add definitions shared with LKM or shared code

[McDaniel, Timothy]

Change-Id: Ie39013936676771d096c2166b2a4745cdeb772b0
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_user.h | 1351 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1351 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h

diff --git a/drivers/event/dlb/dlb_user.h b/drivers/event/dlb/dlb_user.h
new file mode 100644
index 000000000..f2dcee190
--- /dev/null
+++ b/drivers/event/dlb/dlb_user.h
@@ -0,0 +1,1351 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#define DLB_MAX_NAME_LEN 64
+
+#include <linux/types.h>
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' device file commands */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_GET_DEVICE_VERSION: Query the DLB device version.
+ *
+ *	This ioctl interface is the same in all driver versions and is always
+ *	the first ioctl.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id[7:0]: Device revision.
+ *	response.id[15:8]: Device version.
+ */
+
+struct dlb_get_device_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+#define DLB_VERSION_MAJOR_NUMBER 10
+#define DLB_VERSION_MINOR_NUMBER 7
+#define DLB_VERSION_REVISION_NUMBER 9
+#define DLB_VERSION (DLB_VERSION_MAJOR_NUMBER << 24 | \
+		     DLB_VERSION_MINOR_NUMBER << 16 | \
+		     DLB_VERSION_REVISION_NUMBER)
+
+#define DLB_VERSION_GET_MAJOR_NUMBER(x) (((x) >> 24) & 0xFF)
+#define DLB_VERSION_GET_MINOR_NUMBER(x) (((x) >> 16) & 0xFF)
+#define DLB_VERSION_GET_REVISION_NUMBER(x) ((x) & 0xFFFF)
+
+static inline __u8 dlb_version_incompatible(__u32 version)
+{
+	__u8 inc;
+
+	inc = DLB_VERSION_GET_MAJOR_NUMBER(version) != DLB_VERSION_MAJOR_NUMBER;
+	inc |= (int)DLB_VERSION_GET_MINOR_NUMBER(version) <
+		DLB_VERSION_MINOR_NUMBER;
+
+	return inc;
+}
+
+/*
+ * DLB_CMD_GET_DRIVER_VERSION: Query the DLB driver version. The major number
+ *	is changed when there is an ABI-breaking change, the minor number is
+ *	changed if the API is changed in a backwards-compatible way, and the
+ *	revision number is changed for fixes that don't affect the API.
+ *
+ *	If the kernel driver's API version major number and the header's
+ *	DLB_VERSION_MAJOR_NUMBER differ, the two are incompatible, or if the
+ *	major numbers match but the kernel driver's minor number is less than
+ *	the header file's, they are incompatible. The DLB_VERSION_INCOMPATIBLE
+ *	macro should be used to check for compatibility.
+ *
+ *	This ioctl interface is the same in all driver versions. Applications
+ *	should check the driver version before performing any other ioctl
+ *	operations.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Driver API version. Use the DLB_VERSION_GET_MAJOR_NUMBER,
+ *		DLB_VERSION_GET_MINOR_NUMBER, and
+ *		DLB_VERSION_GET_REVISION_NUMBER macros to interpret the field.
+ */
+
+struct dlb_get_driver_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SAMPLE_PERF_COUNTERS: Gather a set of DLB performance data by
+ *	enabling performance counters for a user-specified measurement duration.
+ *	This ioctl is blocking; the calling thread sleeps in the kernel driver
+ *	for the duration of the measurement, then writes the data to user
+ *	memory before returning.
+ *
+ *	Certain metrics cannot be measured simultaneously, so multiple
+ *	invocations of this command are necessary to gather all metrics.
+ *	Metrics that can be collected simultaneously are grouped together in
+ *	struct dlb_perf_metric_group_X.
+ *
+ *	The driver allows only one active measurement at a time. If a thread
+ *	calls this command while a measurement is ongoing, the thread will
+ *	block until the original measurement completes.
+ *
+ *	This ioctl is not supported for VF devices.
+ *
+ * Input parameters:
+ * - measurement_duration_us: Duration, in microseconds, of the
+ *	measurement period. The duration must be between 1us and 60s,
+ *	inclusive.
+ * - perf_metric_group_id: ID of the metric group to measure.
+ * - perf_metric_group_data: Pointer to union dlb_perf_metric_group_data
+ *	structure. The driver will interpret the union according to
+ *	perf_metric_group_ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_perf_metric_group_0 {
+	__u32 dlb_iosf_to_sys_enq_count;
+	__u32 dlb_sys_to_iosf_deq_count;
+	__u32 dlb_sys_to_dlb_enq_count;
+	__u32 dlb_dlb_to_sys_deq_count;
+};
+
+struct dlb_perf_metric_group_1 {
+	__u32 dlb_push_ptr_update_count;
+};
+
+struct dlb_perf_metric_group_2 {
+	__u32 dlb_avg_hist_list_depth;
+};
+
+struct dlb_perf_metric_group_3 {
+	__u32 dlb_avg_qed_depth;
+};
+
+struct dlb_perf_metric_group_4 {
+	__u32 dlb_avg_dqed_depth;
+};
+
+struct dlb_perf_metric_group_5 {
+	__u32 dlb_noop_hcw_count;
+	__u32 dlb_bat_t_hcw_count;
+};
+
+struct dlb_perf_metric_group_6 {
+	__u32 dlb_comp_hcw_count;
+	__u32 dlb_comp_t_hcw_count;
+};
+
+struct dlb_perf_metric_group_7 {
+	__u32 dlb_enq_hcw_count;
+	__u32 dlb_enq_t_hcw_count;
+};
+
+struct dlb_perf_metric_group_8 {
+	__u32 dlb_renq_hcw_count;
+	__u32 dlb_renq_t_hcw_count;
+};
+
+struct dlb_perf_metric_group_9 {
+	__u32 dlb_rel_hcw_count;
+};
+
+struct dlb_perf_metric_group_10 {
+	__u32 dlb_frag_hcw_count;
+	__u32 dlb_frag_t_hcw_count;
+};
+
+union dlb_perf_metric_group_data {
+	struct dlb_perf_metric_group_0 group_0;
+	struct dlb_perf_metric_group_1 group_1;
+	struct dlb_perf_metric_group_2 group_2;
+	struct dlb_perf_metric_group_3 group_3;
+	struct dlb_perf_metric_group_4 group_4;
+	struct dlb_perf_metric_group_5 group_5;
+	struct dlb_perf_metric_group_6 group_6;
+	struct dlb_perf_metric_group_7 group_7;
+	struct dlb_perf_metric_group_8 group_8;
+	struct dlb_perf_metric_group_9 group_9;
+	struct dlb_perf_metric_group_10 group_10;
+};
+
+struct dlb_sample_perf_counters_args {
+	/* Output parameters */
+	__u64 elapsed_time_us;
+	__u64 response;
+	/* Input parameters */
+	__u32 measurement_duration_us;
+	__u32 perf_metric_group_id;
+	__u64 perf_metric_group_data;
+};
+
+/*
+ * DLB_CMD_MEASURE_SCHED_COUNTS: Measure the DLB scheduling activity for a
+ *	user-specified measurement duration. This ioctl is blocking; the
+ *	calling thread sleeps in the kernel driver for the duration of the
+ *	measurement, then writes the result to user memory before returning.
+ *
+ *	Unlike the DLB_CMD_SAMPLE_PERF_COUNTERS ioctl, multiple threads can
+ *	measure scheduling counts simultaneously.
+ *
+ *	Note: VF devices can only measure the scheduling counts of their CQs;
+ *	all other counts will be set to 0.
+ *
+ * Input parameters:
+ * - measurement_duration_us: Duration, in microseconds, of the
+ *	measurement period. The duration must be between 1us and 60s,
+ *	inclusive.
+ * - padding0: Reserved for future use.
+ * - sched_count_data: Pointer to a struct dlb_sched_count data structure.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_sched_counts {
+	__u64 ldb_sched_count;
+	__u64 dir_sched_count;
+	__u64 ldb_cq_sched_count[64];
+	__u64 dir_cq_sched_count[128];
+};
+
+struct dlb_measure_sched_count_args {
+	/* Output parameters */
+	__u64 elapsed_time_us;
+	__u64 response;
+	/* Input parameters */
+	__u32 measurement_duration_us;
+	__u32 padding0;
+	__u64 sched_count_data;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_user_interface_commands {
+	DLB_CMD_GET_DEVICE_VERSION,
+	DLB_CMD_CREATE_SCHED_DOMAIN,
+	DLB_CMD_GET_NUM_RESOURCES,
+	DLB_CMD_GET_DRIVER_VERSION,
+	DLB_CMD_SAMPLE_PERF_COUNTERS,
+	DLB_CMD_SET_SN_ALLOCATION,
+	DLB_CMD_GET_SN_ALLOCATION,
+	DLB_CMD_MEASURE_SCHED_COUNTS,
+	DLB_CMD_QUERY_CQ_POLL_MODE,
+	DLB_CMD_GET_SN_OCCUPANCY,
+
+	/* NUM_DLB_CMD must be last */
+	NUM_DLB_CMD,
+};
+
+/*******************************/
+/* 'domain' device file alerts */
+/*******************************/
+
+/* Scheduling domain device files can be read to receive domain-specific
+ * notifications, for alerts such as hardware errors.
+ *
+ * Each alert is encoded in a 16B message. The first 8B contains the alert ID,
+ * and the second 8B is optional and contains additional information.
+ * Applications should cast read data to a struct dlb_domain_alert, and
+ * interpret the struct's alert_id according to dlb_domain_alert_id. The read
+ * length must be 16B, or the function will return -EINVAL.
+ *
+ * Reads are destructive, and in the case of multiple file descriptors for the
+ * same domain device file, an alert will be read by only one of the file
+ * descriptors.
+ *
+ * The driver stores alerts in a fixed-size alert ring until they are read. If
+ * the alert ring fills completely, subsequent alerts will be dropped. It is
+ * recommended that DLB applications dedicate a thread to perform blocking
+ * reads on the device file.
+ */
+enum dlb_domain_alert_id {
+	/* A destination domain queue that this domain connected to has
+	 * unregistered, and can no longer be sent to. The aux alert data
+	 * contains the queue ID.
+	 */
+	DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER,
+	/* A producer port in this domain attempted to send a QE without a
+	 * credit. aux_alert_data[7:0] contains the port ID, and
+	 * aux_alert_data[15:8] contains a flag indicating whether the port is
+	 * load-balanced (1) or directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS,
+	/* Software issued an illegal enqueue for a port in this domain. An
+	 * illegal enqueue could be:
+	 * - Illegal (excess) completion
+	 * - Illegal fragment
+	 * - Illegal enqueue command
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ,
+	/* Software issued excess CQ token pops for a port in this domain.
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS,
+	/* A enqueue contained either an invalid command encoding or a REL,
+	 * REL_T, RLS, FWD, FWD_T, FRAG, or FRAG_T from a directed port.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_HCW,
+	/* The QID must be valid and less than 128.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_QID,
+	/* An enqueue went to a disabled QID.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_DISABLED_QID,
+	/* The device containing this domain was reset. All applications using
+	 * the device need to exit for the driver to complete the reset
+	 * procedure.
+	 *
+	 * aux_alert_data doesn't contain any information for this alert.
+	 */
+	DLB_DOMAIN_ALERT_DEVICE_RESET,
+	/* User-space has enqueued an alert.
+	 *
+	 * aux_alert_data contains user-provided data.
+	 */
+	DLB_DOMAIN_ALERT_USER,
+
+	/* Number of DLB domain alerts */
+	NUM_DLB_DOMAIN_ALERTS
+};
+
+static const char dlb_domain_alert_strings[][128] = {
+	"DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER",
+	"DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS",
+	"DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ",
+	"DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS",
+	"DLB_DOMAIN_ALERT_ILLEGAL_HCW",
+	"DLB_DOMAIN_ALERT_ILLEGAL_QID",
+	"DLB_DOMAIN_ALERT_DISABLED_QID",
+	"DLB_DOMAIN_ALERT_DEVICE_RESET",
+	"DLB_DOMAIN_ALERT_USER",
+};
+
+struct dlb_domain_alert {
+	__u64 alert_id;
+	__u64 aux_alert_data;
+};
+
+/*********************************/
+/* 'domain' device file commands */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT: Block on a CQ interrupt until a QE
+ *	arrives for the specified port. If a QE is already present, the ioctl
+ *	will immediately return.
+ *
+ *	Note: Only one thread can block on a CQ's interrupt at a time. Doing
+ *	otherwise can result in hung threads.
+ *
+ * Input parameters:
+ * - port_id: Port ID.
+ * - is_ldb: True if the port is load-balanced, false otherwise.
+ * - arm: Tell the driver to arm the interrupt.
+ * - cq_gen: Current CQ generation bit.
+ * - padding0: Reserved for future use.
+ * - cq_va: VA of the CQ entry where the next QE will be placed.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_block_on_cq_interrupt_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u8 is_ldb;
+	__u8 arm;
+	__u8 cq_gen;
+	__u8 padding0;
+	__u64 cq_va;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT: Enqueue a domain alert that will be
+ *	read by one reader thread.
+ *
+ * Input parameters:
+ * - aux_alert_data: user-defined auxiliary data.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enqueue_domain_alert_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u64 aux_alert_data;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+enum dlb_domain_user_interface_commands {
+	DLB_DOMAIN_CMD_CREATE_LDB_POOL,
+	DLB_DOMAIN_CMD_CREATE_DIR_POOL,
+	DLB_DOMAIN_CMD_CREATE_LDB_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_DIR_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_LDB_PORT,
+	DLB_DOMAIN_CMD_CREATE_DIR_PORT,
+	DLB_DOMAIN_CMD_START_DOMAIN,
+	DLB_DOMAIN_CMD_MAP_QID,
+	DLB_DOMAIN_CMD_UNMAP_QID,
+	DLB_DOMAIN_CMD_ENABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_ENABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_DISABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_DISABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT,
+	DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT,
+	DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS,
+
+	/* NUM_DLB_DOMAIN_CMD must be last */
+	NUM_DLB_DOMAIN_CMD,
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+/*******************/
+/* dlb ioctl codes */
+/*******************/
+
+#define DLB_IOC_MAGIC  'h'
+
+#define DLB_IOC_GET_DEVICE_VERSION				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_GET_DEVICE_VERSION,		\
+		      struct dlb_get_driver_version_args)
+#define DLB_IOC_CREATE_SCHED_DOMAIN				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_CREATE_SCHED_DOMAIN,		\
+		      struct dlb_create_sched_domain_args)
+#define DLB_IOC_GET_NUM_RESOURCES				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_GET_NUM_RESOURCES,		\
+		      struct dlb_get_num_resources_args)
+#define DLB_IOC_GET_DRIVER_VERSION				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_GET_DRIVER_VERSION,		\
+		      struct dlb_get_driver_version_args)
+#define DLB_IOC_SAMPLE_PERF_COUNTERS				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_SAMPLE_PERF_COUNTERS,		\
+		      struct dlb_sample_perf_counters_args)
+#define DLB_IOC_SET_SN_ALLOCATION				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_SET_SN_ALLOCATION,		\
+		      struct dlb_set_sn_allocation_args)
+#define DLB_IOC_GET_SN_ALLOCATION				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_GET_SN_ALLOCATION,		\
+		      struct dlb_get_sn_allocation_args)
+#define DLB_IOC_MEASURE_SCHED_COUNTS				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_MEASURE_SCHED_COUNTS,		\
+		      struct dlb_measure_sched_count_args)
+#define DLB_IOC_QUERY_CQ_POLL_MODE				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_QUERY_CQ_POLL_MODE,		\
+		      struct dlb_query_cq_poll_mode_args)
+#define DLB_IOC_GET_SN_OCCUPANCY				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_CMD_GET_SN_OCCUPANCY,			\
+		      struct dlb_get_sn_occupancy_args)
+#define DLB_IOC_CREATE_LDB_POOL					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_LDB_POOL,		\
+		      struct dlb_create_ldb_pool_args)
+#define DLB_IOC_CREATE_DIR_POOL					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_DIR_POOL,		\
+		      struct dlb_create_dir_pool_args)
+#define DLB_IOC_CREATE_LDB_QUEUE				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_LDB_QUEUE,		\
+		      struct dlb_create_ldb_queue_args)
+#define DLB_IOC_CREATE_DIR_QUEUE				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_DIR_QUEUE,		\
+		      struct dlb_create_dir_queue_args)
+#define DLB_IOC_CREATE_LDB_PORT					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_LDB_PORT,		\
+		      struct dlb_create_ldb_port_args)
+#define DLB_IOC_CREATE_DIR_PORT					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_CREATE_DIR_PORT,		\
+		      struct dlb_create_dir_port_args)
+#define DLB_IOC_START_DOMAIN					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_START_DOMAIN,		\
+		      struct dlb_start_domain_args)
+#define DLB_IOC_MAP_QID						\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_MAP_QID,			\
+		      struct dlb_map_qid_args)
+#define DLB_IOC_UNMAP_QID					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_UNMAP_QID,			\
+		      struct dlb_unmap_qid_args)
+#define DLB_IOC_ENABLE_LDB_PORT					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_ENABLE_LDB_PORT,		\
+		      struct dlb_enable_ldb_port_args)
+#define DLB_IOC_ENABLE_DIR_PORT					\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_ENABLE_DIR_PORT,		\
+		      struct dlb_enable_dir_port_args)
+#define DLB_IOC_DISABLE_LDB_PORT				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_DISABLE_LDB_PORT,		\
+		      struct dlb_disable_ldb_port_args)
+#define DLB_IOC_DISABLE_DIR_PORT				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_DISABLE_DIR_PORT,		\
+		      struct dlb_disable_dir_port_args)
+#define DLB_IOC_BLOCK_ON_CQ_INTERRUPT				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT,	\
+		      struct dlb_block_on_cq_interrupt_args)
+#define DLB_IOC_ENQUEUE_DOMAIN_ALERT				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT,	\
+		      struct dlb_enqueue_domain_alert_args)
+#define DLB_IOC_GET_LDB_QUEUE_DEPTH				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH,	\
+		      struct dlb_get_ldb_queue_depth_args)
+#define DLB_IOC_GET_DIR_QUEUE_DEPTH				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH,	\
+		      struct dlb_get_dir_queue_depth_args)
+#define DLB_IOC_PENDING_PORT_UNMAPS				\
+		_IOWR(DLB_IOC_MAGIC,				\
+		      DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS,	\
+		      struct dlb_pending_port_unmaps_args)
+
+#endif /* __DLB_USER_H */
-- 
2.13.6

[dpdk-dev] [PATCH 09/27] event/dlb: inline functions used in multiple files

[McDaniel, Timothy]

Change-Id: I61196d9c19cd2add26b6600fb42588ef523febac
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_inline_fns.h | 80 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h

diff --git a/drivers/event/dlb/dlb_inline_fns.h b/drivers/event/dlb/dlb_inline_fns.h
new file mode 100644
index 000000000..0f036f6ee
--- /dev/null
+++ b/drivers/event/dlb/dlb_inline_fns.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file.
+ */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq(struct dlb_enqueue_qe *qe4, uint64_t *pp_addr)
+{
+	/* Move entire 64B cache line of QEs, 128 bits (16B) at a time. */
+	long long *_qe  = (long long *)qe4;
+	__v2di src_data0 = (__v2di){_qe[0], _qe[1]};
+	__v2di src_data1 = (__v2di){_qe[2], _qe[3]};
+	__v2di src_data2 = (__v2di){_qe[4], _qe[5]};
+	__v2di src_data3 = (__v2di){_qe[6], _qe[7]};
+
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 0, (__v2di)src_data0);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 1, (__v2di)src_data1);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 2, (__v2di)src_data2);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 3, (__v2di)src_data3);
+	rte_wmb();
+}
+
+static inline void
+dlb_movntdq_single(void *qe4, void *pp_addr)
+{
+	long long *_qe  = (long long *)qe4;
+	__v2di src_data0 = (__v2di){_qe[0], _qe[1]};
+
+	__builtin_ia32_movntdq((__v2di *)pp_addr, (__v2di)src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(struct dlb_enqueue_qe *qe4, uint64_t *pp_addr)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (pp_addr), "d" (qe4));
+}
-- 
2.13.6

[dpdk-dev] [PATCH 10/27] event/dlb: add PFPMD-specific interface layer to shared code

[McDaniel, Timothy]

Change-Id: I8064d21dfa56df9a797e22f116fc7a3b15b1494c
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/pf/dlb_main.c | 670 ++++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.h |  90 +++++
 drivers/event/dlb/pf/dlb_pf.c   | 839 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 1599 insertions(+)
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c

diff --git a/drivers/event/dlb/pf/dlb_main.c b/drivers/event/dlb/pf/dlb_main.c
new file mode 100644
index 000000000..c5aa16e1c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.c
@@ -0,0 +1,670 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+
+#include "base/dlb_resource.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_regs.h"
+#include "dlb_main.h"
+#include "../dlb_user.h"
+
+
+#define PF_ID_ZERO 0	/* PF ONLY! */
+#define NO_OWNER_VF 0	/* PF ONLY! */
+#define NOT_VF_REQ false /* PF ONLY! */
+
+unsigned int dlb_unregister_timeout_s = DLB_DEFAULT_UNREGISTER_TIMEOUT_S;
+
+#define DLB_PCI_CFG_SPACE_SIZE 256
+#define DLB_PCI_CAP_POINTER 0x34
+#define DLB_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
+#define DLB_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
+#define DLB_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
+#define DLB_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
+#define DLB_PCI_EXT_CAP_ID_ERR 1
+#define DLB_PCI_ERR_UNCOR_MASK 8
+#define DLB_PCI_ERR_UNC_UNSUP  0x00100000
+
+#define DLB_PCI_EXP_DEVCTL 8
+#define DLB_PCI_LNKCTL 16
+#define DLB_PCI_SLTCTL 24
+#define DLB_PCI_RTCTL 28
+#define DLB_PCI_EXP_DEVCTL2 40
+#define DLB_PCI_LNKCTL2 48
+#define DLB_PCI_SLTCTL2 56
+#define DLB_PCI_CMD 4
+#define DLB_PCI_X_CMD 2
+#define DLB_PCI_EXP_DEVSTA 10
+#define DLB_PCI_EXP_DEVSTA_TRPND 0x20
+#define DLB_PCI_EXP_DEVCTL_BCR_FLR 0x8000
+#define DLB_PCI_PASID_CTRL 6
+#define DLB_PCI_PASID_CAP 4
+
+#define DLB_PCI_CAP_ID_EXP       0x10
+#define DLB_PCI_CAP_ID_MSIX      0x11
+#define DLB_PCI_EXT_CAP_ID_PAS   0x1B
+#define DLB_PCI_EXT_CAP_ID_PRI   0x13
+#define DLB_PCI_EXT_CAP_ID_ACS   0xD
+
+#define DLB_PCI_PASID_CAP_EXEC          0x2
+#define DLB_PCI_PASID_CAP_PRIV          0x4
+#define DLB_PCI_PASID_CTRL_ENABLE       0x1
+#define DLB_PCI_PRI_CTRL_ENABLE         0x1
+#define DLB_PCI_PRI_ALLOC_REQ           0xC
+#define DLB_PCI_PRI_CTRL                0x4
+#define DLB_PCI_MSIX_FLAGS              0x2
+#define DLB_PCI_MSIX_FLAGS_ENABLE       0x8000
+#define DLB_PCI_MSIX_FLAGS_MASKALL      0x4000
+#define DLB_PCI_ERR_ROOT_STATUS         0x30
+#define DLB_PCI_ERR_COR_STATUS          0x10
+#define DLB_PCI_ERR_UNCOR_STATUS        0x4
+#define DLB_PCI_COMMAND_INTX_DISABLE    0x400
+#define DLB_PCI_ACS_CAP                 0x4
+#define DLB_PCI_ACS_CTRL                0x6
+#define DLB_PCI_ACS_SV                  0x1
+#define DLB_PCI_ACS_RR                  0x4
+#define DLB_PCI_ACS_CR                  0x8
+#define DLB_PCI_ACS_UF                  0x10
+#define DLB_PCI_ACS_EC                  0x20
+
+static inline void
+dlb_movntdq(void *qe4, void *pp_addr)
+{
+	/* Move entire 64B cache line of QEs, 128 bits (16B) at a time. */
+	long long *_qe  = (long long *)qe4;
+	__v2di src_data0 = (__v2di){_qe[0], _qe[1]};
+	__v2di src_data1 = (__v2di){_qe[2], _qe[3]};
+	__v2di src_data2 = (__v2di){_qe[4], _qe[5]};
+	__v2di src_data3 = (__v2di){_qe[6], _qe[7]};
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 0, (__v2di)src_data0);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 1, (__v2di)src_data1);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 2, (__v2di)src_data2);
+	rte_wmb();
+	__builtin_ia32_movntdq((__v2di *)pp_addr + 3, (__v2di)src_data3);
+	rte_wmb();
+}
+
+static inline void
+dlb_movdir64b(void *qe4, void *pp_addr)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		:
+		: "a" (pp_addr), "d" (qe4));
+}
+
+
+static int dlb_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint32_t hdr;
+	size_t sz;
+	int pos;
+
+	pos = DLB_PCI_CFG_SPACE_SIZE;
+	sz = sizeof(hdr);
+
+	while (pos > 0xFF) {
+		if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
+			return -1;
+
+		if (DLB_PCI_EXT_CAP_ID(hdr) == id)
+			return pos;
+
+		pos = DLB_PCI_EXT_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
+{
+	uint8_t pos;
+	int ret;
+	uint16_t hdr;
+
+	ret = rte_pci_read_config(pdev, &pos, 1, DLB_PCI_CAP_POINTER);
+	pos &= 0xFC;
+
+	if (ret != 1)
+		return -1;
+
+	while (pos > 0x3F) {
+		ret = rte_pci_read_config(pdev, &hdr, 2, pos);
+		if (ret != 2)
+			return -1;
+
+		if (DLB_PCI_CAP_ID(hdr) == id)
+			return pos;
+
+		if (DLB_PCI_CAP_ID(hdr) == 0xFF)
+			return -1;
+
+		pos = DLB_PCI_CAP_NEXT(hdr);
+	}
+
+	return -1;
+}
+
+static int dlb_mask_ur_err(struct rte_pci_device *pdev)
+{
+	uint32_t mask;
+	size_t sz = sizeof(mask);
+	int pos = dlb_pci_find_ext_capability(pdev, DLB_PCI_EXT_CAP_ID_ERR);
+
+	if (pos < 0) {
+		printf("[%s()] failed to find the aer capability\n",
+		       __func__);
+		return pos;
+	}
+
+	pos += DLB_PCI_ERR_UNCOR_MASK;
+
+	if (rte_pci_read_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to read uncorrectable error mask reg\n",
+		       __func__);
+		return -1;
+	}
+
+	/* Mask Unsupported Request errors */
+	mask |= DLB_PCI_ERR_UNC_UNSUP;
+
+	if (rte_pci_write_config(pdev, &mask, sz, pos) != (int)sz) {
+		printf("[%s()] Failed to write uncorrectable error mask reg at offset %d\n",
+		       __func__, pos);
+		return -1;
+	}
+
+	return 0;
+}
+
+struct dlb_dev *
+dlb_probe(struct rte_pci_device *pdev)
+{
+	struct dlb_dev *dlb_dev;
+	int ret = 0;
+
+	DLB_INFO(dlb_dev, "probe\n");
+
+	dlb_dev = rte_malloc("DLB_PF", sizeof(struct dlb_dev),
+			     RTE_CACHE_LINE_SIZE);
+
+	if (!dlb_dev) {
+		ret = -ENOMEM;
+		goto dlb_dev_malloc_fail;
+	}
+
+	/* PCI Bus driver has already mapped bar space into process.
+	 * Save off our IO register and FUNC addresses.
+	 */
+
+	/* BAR 0 */
+	if (pdev->mem_resource[0].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
+	dlb_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB FUNC VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.func_kva,
+		 (void *)dlb_dev->hw.func_phys_addr,
+		 pdev->mem_resource[0].len);
+
+	/* BAR 2 */
+	if (pdev->mem_resource[2].addr == NULL) {
+		DLB_ERR(dlb_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
+		ret = -EINVAL;
+		goto pci_mmap_bad_addr;
+	}
+	dlb_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
+	dlb_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
+
+	DLB_INFO(dlb_dev, "DLB CSR VA=%p, PA=%p, len=%"PRIu64"\n",
+		 (void *)dlb_dev->hw.csr_kva,
+		 (void *)dlb_dev->hw.csr_phys_addr,
+		 pdev->mem_resource[2].len);
+
+	dlb_dev->pdev = pdev;
+
+	ret = dlb_pf_reset(dlb_dev);
+	if (ret)
+		goto dlb_reset_fail;
+
+	/* DLB incorrectly sends URs in response to certain messages. Mask UR
+	 * errors to prevent these from being propagated to the MCA.
+	 */
+	ret = dlb_mask_ur_err(pdev);
+	if (ret)
+		goto mask_ur_err_fail;
+
+	ret = dlb_pf_init_interrupts(dlb_dev);
+	if (ret)
+		goto init_interrupts_fail;
+
+	ret = dlb_pf_init_driver_state(dlb_dev);
+	if (ret)
+		goto init_driver_state_fail;
+
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
+	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
+
+	dlb_pf_init_hardware(dlb_dev);
+
+	return dlb_dev;
+
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
+init_driver_state_fail:
+	dlb_pf_free_interrupts(dlb_dev);
+init_interrupts_fail:
+mask_ur_err_fail:
+dlb_reset_fail:
+pci_mmap_bad_addr:
+	rte_free(dlb_dev);
+dlb_dev_malloc_fail:
+	rte_errno = ret;
+	return NULL;
+}
+
+int
+dlb_pf_reset(struct dlb_dev *dlb_dev)
+{
+	int ret = 0;
+	int i = 0;
+	uint32_t dword[16];
+	uint16_t cmd;
+	off_t off;
+
+	uint16_t dev_ctl_word;
+	uint16_t dev_ctl2_word;
+	uint16_t lnk_word;
+	uint16_t lnk_word2;
+	uint16_t slt_word;
+	uint16_t slt_word2;
+	uint16_t rt_ctl_word;
+	uint32_t pri_reqs_dword;
+	uint16_t pri_ctrl_word;
+
+	int pcie_cap_offset;
+	int pasid_cap_offset;
+	int pri_cap_offset;
+	int msix_cap_offset;
+	int err_cap_offset;
+	int acs_cap_offset;
+	int wait_count;
+
+	uint16_t devsta_busy_word;
+	uint16_t devctl_word;
+	uint16_t pasid_ctrl_word;
+	uint16_t pasid_features;
+
+	struct rte_pci_device *pdev = dlb_dev->pdev;
+
+	/* Save PCI config state */
+
+	for (i = 0; i < 16; i++) {
+		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
+			return ret;
+	}
+
+	pcie_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_EXP);
+
+	if (pcie_cap_offset < 0) {
+		printf("[%s()] failed to find the pcie capability\n",
+		       __func__);
+		return pcie_cap_offset;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
+		dev_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL;
+	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
+		lnk_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL;
+	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
+		slt_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_RTCTL;
+	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
+		rt_ctl_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
+		dev_ctl2_word = 0;
+
+	off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
+		lnk_word2 = 0;
+
+	off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
+		slt_word2 = 0;
+
+	pri_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_PRI);
+	if (pri_cap_offset >= 0) {
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
+			pri_reqs_dword = 0;
+	}
+
+	/* clear the PCI command register before issuing the FLR */
+
+	off = DLB_PCI_CMD;
+	cmd = 0;
+	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+		printf("[%s()] failed to write pci config space at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	/* issue the FLR */
+	for (wait_count = 0; wait_count < 4; wait_count++) {
+		int sleep_time;
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVSTA;
+		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
+		if (ret != 2) {
+			printf("[%s()] failed to read the pci device status\n",
+			       __func__);
+			return ret;
+		}
+
+		if (!(devsta_busy_word & DLB_PCI_EXP_DEVSTA_TRPND))
+			break;
+
+		sleep_time = (1 << (wait_count)) * 100;
+		rte_delay_ms(sleep_time);
+	}
+
+	if (wait_count == 4) {
+		printf("[%s()] wait for pci pending transactions timed out\n",
+		       __func__);
+		return -1;
+	}
+
+	off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
+	if (ret != 2) {
+		printf("[%s()] failed to read the pcie device control\n",
+		       __func__);
+		return ret;
+	}
+
+	devctl_word |= DLB_PCI_EXP_DEVCTL_BCR_FLR;
+
+	if (rte_pci_write_config(pdev, &devctl_word, 2, off) != 2) {
+		printf("[%s()] failed to write the pcie device control at offset %d\n",
+		       __func__, (int)off);
+		return -1;
+	}
+
+	rte_delay_ms(100);
+
+	/* Restore PCI config state */
+
+	if (pcie_cap_offset >= 0) {
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL;
+		if (rte_pci_write_config(pdev, &dev_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write the pcie device control at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL;
+		if (rte_pci_write_config(pdev, &lnk_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL;
+		if (rte_pci_write_config(pdev, &slt_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_RTCTL;
+		if (rte_pci_write_config(pdev, &rt_ctl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_EXP_DEVCTL2;
+		if (rte_pci_write_config(pdev, &dev_ctl2_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_LNKCTL2;
+		if (rte_pci_write_config(pdev, &lnk_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pcie_cap_offset + DLB_PCI_SLTCTL2;
+		if (rte_pci_write_config(pdev, &slt_word2, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	pasid_cap_offset = dlb_pci_find_ext_capability(pdev,
+						       DLB_PCI_EXT_CAP_ID_PAS);
+	if (pasid_cap_offset >= 0) {
+		off = pasid_cap_offset + DLB_PCI_PASID_CAP;
+		if (rte_pci_read_config(pdev, &pasid_features, 2, off) != 2)
+			pasid_features = 0;
+
+		pasid_features &= DLB_PCI_PASID_CAP_EXEC;
+		pasid_features &= DLB_PCI_PASID_CAP_PRIV;
+		pasid_ctrl_word = DLB_PCI_PASID_CTRL_ENABLE | pasid_features;
+
+		off = pasid_cap_offset + DLB_PCI_PASID_CTRL;
+		if (rte_pci_write_config(pdev, &pasid_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	if (pri_cap_offset >= 0) {
+		pri_ctrl_word = DLB_PCI_PRI_CTRL_ENABLE;
+
+		off = pri_cap_offset + DLB_PCI_PRI_ALLOC_REQ;
+		if (rte_pci_write_config(pdev, &pri_reqs_dword, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = pri_cap_offset + DLB_PCI_PRI_CTRL;
+		if (rte_pci_write_config(pdev, &pri_ctrl_word, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	err_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ERR);
+	if (err_cap_offset >= 0) {
+		uint32_t tmp;
+
+		off = err_cap_offset + DLB_PCI_ERR_ROOT_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_COR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = err_cap_offset + DLB_PCI_ERR_UNCOR_STATUS;
+		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
+			tmp = 0;
+
+		if (rte_pci_write_config(pdev, &tmp, 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	for (i = 16; i > 0; i--) {
+		off = (i - 1) * 4;
+		if (rte_pci_write_config(pdev, &dword[i - 1], 4, off) != 4) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	off = DLB_PCI_CMD;
+	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+		cmd &= ~DLB_PCI_COMMAND_INTX_DISABLE;
+		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space\n",
+			       __func__);
+			return -1;
+		}
+	}
+
+	msix_cap_offset = dlb_pci_find_capability(pdev, DLB_PCI_CAP_ID_MSIX);
+	if (msix_cap_offset >= 0) {
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd |= DLB_PCI_MSIX_FLAGS_ENABLE;
+			cmd |= DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+
+		off = msix_cap_offset + DLB_PCI_MSIX_FLAGS;
+		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
+			cmd &= ~DLB_PCI_MSIX_FLAGS_MASKALL;
+			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
+				printf("[%s()] failed to write msix flags\n",
+				       __func__);
+				return -1;
+			}
+		}
+	}
+
+	acs_cap_offset = dlb_pci_find_ext_capability(pdev,
+						     DLB_PCI_EXT_CAP_ID_ACS);
+	if (acs_cap_offset >= 0) {
+		uint16_t acs_cap, acs_ctrl, acs_mask;
+		off = acs_cap_offset + DLB_PCI_ACS_CAP;
+		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
+			acs_cap = 0;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_SV | DLB_PCI_ACS_RR;
+		acs_mask |= (DLB_PCI_ACS_CR | DLB_PCI_ACS_UF);
+		acs_ctrl |= (acs_cap & acs_mask);
+
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
+			acs_ctrl = 0;
+
+		acs_mask = DLB_PCI_ACS_RR | DLB_PCI_ACS_CR | DLB_PCI_ACS_EC;
+		acs_ctrl &= ~acs_mask;
+
+		off = acs_cap_offset + DLB_PCI_ACS_CTRL;
+		if (rte_pci_write_config(pdev, &acs_ctrl, 2, off) != 2) {
+			printf("[%s()] failed to write pci config space at offset %d\n",
+			       __func__, (int)off);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*******************************/
+/****** Driver management ******/
+/*******************************/
+
+int
+dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
+{
+	int i;
+
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_MOVDIR64B))
+		dlb_dev->enqueue_four = dlb_movdir64b;
+	else
+		dlb_dev->enqueue_four = dlb_movntdq;
+
+	/* Initialize software state */
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		dlb_list_init_head(&dlb_dev->ldb_port_pages[i].list);
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		dlb_list_init_head(&dlb_dev->dir_port_pages[i].list);
+
+	rte_spinlock_init(&dlb_dev->resource_mutex);
+	rte_spinlock_init(&dlb_dev->measurement_lock);
+
+	return 0;
+}
+
+void
+dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
+{
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+}
diff --git a/drivers/event/dlb/pf/dlb_main.h b/drivers/event/dlb/pf/dlb_main.h
new file mode 100644
index 000000000..d717b933c
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_main.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_MAIN_H
+#define __DLB_MAIN_H
+
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+#endif
+
+#include "base/dlb_hw_types.h"
+#include "../dlb_user.h"
+
+#define DLB_DEFAULT_UNREGISTER_TIMEOUT_S 5
+
+struct dlb_dev;
+
+struct dlb_port_page {
+	struct dlb_list_head list;
+	unsigned long offs;
+	struct iova *iova;
+	struct page *page;
+	uintptr_t dma_addr;
+	unsigned long dma_size;
+	int refcnt;
+};
+
+struct dlb_port_memory {
+	struct dlb_list_head list;
+	struct dlb_port_page *pages;
+	void *cq_base;
+	uintptr_t cq_dma_base;
+	void *pc_base;
+	uintptr_t pc_dma_base;
+	int domain_id;
+	bool valid;
+};
+
+struct dlb_dev {
+	struct rte_pci_device *pdev;
+	struct dlb_hw hw;
+	/* struct list_head list; */
+	struct device *dlb_device;
+	struct dlb_port_memory ldb_port_pages[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_port_memory dir_port_pages[DLB_MAX_NUM_DIR_PORTS];
+	/* The enqueue_four function enqueues four HCWs (one cache-line worth)
+	 * to the DLB, using whichever mechanism is supported by the platform
+	 * on which this driver is running.
+	 */
+	void (*enqueue_four)(void *qe4, void *pp_addr);
+	bool domain_reset_failed;
+	/* The resource mutex serializes access to driver data structures and
+	 * hardware registers.
+	 */
+	rte_spinlock_t resource_mutex;
+	rte_spinlock_t measurement_lock;
+	bool worker_launched;
+	u8 revision;
+};
+
+struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+void dlb_reset_done(struct dlb_dev *dlb_dev);
+
+/* pf_ops */
+int dlb_pf_init_driver_state(struct dlb_dev *dev);
+void dlb_pf_free_driver_state(struct dlb_dev *dev);
+int dlb_pf_init_interrupts(struct dlb_dev *dev);
+int dlb_hw_enable_ldb_cq_interrupts(struct dlb_dev *dev,
+				    int port_id,
+				    u16 thresh);
+int dlb_hw_enable_dir_cq_interrupts(struct dlb_dev *dev,
+				    int port_id,
+				    u16 thresh);
+int dlb_pf_arm_cq_interrupt(struct dlb_dev *dev,
+			    int domain_id,
+			    int port_id,
+			    bool is_ldb);
+void dlb_pf_reinit_interrupts(struct dlb_dev *dev);
+void dlb_pf_free_interrupts(struct dlb_dev *dev);
+void dlb_pf_init_hardware(struct dlb_dev *dev);
+int dlb_pf_reset(struct dlb_dev *dlb_dev);
+
+#endif /* __DLB_MAIN_H */
diff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c
new file mode 100644
index 000000000..0137080c1
--- /dev/null
+++ b/drivers/event/dlb/pf/dlb_pf.c
@@ -0,0 +1,839 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_devargs.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_memory.h>
+#include <rte_string_fns.h>
+
+#include "../dlb_priv.h"
+#include "../dlb_iface.h"
+#include "../dlb_inline_fns.h"
+#include "dlb_main.h"
+#include "base/dlb_hw_types.h"
+#include "base/dlb_osdep.h"
+#include "base/dlb_resource.h"
+
+extern struct dlb_dev *dlb_probe(struct rte_pci_device *pdev);
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+static const char *event_dlb_pf_name = "dlb_event"; /* Same as vdev name */
+
+static void
+dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
+{
+	int i;
+
+	/* Addresses will be initialized at port create */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		/* First directed ports */
+
+		/* producer port */
+		dlb_port[i][DLB_DIR].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_DIR].ldb_popcount = NULL;
+		dlb_port[i][DLB_DIR].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_DIR].cq_base = NULL;
+		dlb_port[i][DLB_DIR].mmaped = true;
+
+		/* Now load balanced ports */
+
+		/* producer port */
+		dlb_port[i][DLB_LDB].pp_addr = NULL;
+
+		/* popcount */
+		dlb_port[i][DLB_LDB].ldb_popcount = NULL;
+		dlb_port[i][DLB_LDB].dir_popcount = NULL;
+
+		/* consumer queue */
+		dlb_port[i][DLB_LDB].cq_base = NULL;
+		dlb_port[i][DLB_LDB].mmaped = true;
+	}
+}
+
+static int
+dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+static void
+dlb_pf_domain_close(struct dlb_eventdev *dlb)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
+	int ret;
+
+	ret = dlb_reset_domain(&dlb_dev->hw,
+			       dlb->qm_instance.domain_id,
+			       false,
+			       0);
+	if (ret)
+		DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
+}
+
+static int
+dlb_pf_get_device_version(struct dlb_hw_dev *handle,
+			  uint8_t *revision)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	*revision = dlb_dev->revision;
+
+	return 0;
+}
+
+#define PF_ID_ZERO 0	/* PF ONLY! */
+#define NO_OWNER_VF 0	/* PF ONLY! */
+#define NOT_VF_REQ false /* PF ONLY! */
+
+static int
+dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
+			 struct dlb_get_num_resources_args *rsrcs)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	return dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs, false, 0);
+}
+
+static int
+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
+			   struct dlb_create_sched_domain_args *arg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (dlb_dev->domain_reset_failed) {
+		response.status = DLB_ST_DOMAIN_RESET_FAILED;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response,
+					 NOT_VF_REQ, PF_ID_ZERO);
+	if (ret)
+		goto done;
+
+done:
+
+	*(struct dlb_cmd_response *)arg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_ldb_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response,
+				     NOT_VF_REQ,
+				     PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
+			      struct dlb_create_dir_pool_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     &response,
+				     NOT_VF_REQ,
+				     PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_ldb_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response,
+				      NOT_VF_REQ,
+				      PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_queue_create(struct dlb_hw_dev *handle,
+			struct dlb_create_dir_queue_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_create_dir_queue(&dlb_dev->hw,
+				      handle->domain_id,
+				      cfg,
+				      &response,
+				      NOT_VF_REQ,
+				      PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static void *
+dlb_alloc_coherent_aligned(rte_iova_t *phys, size_t size, int align)
+{
+	const struct rte_memzone *mz;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t core_id = rte_lcore_id();
+	unsigned int socket_id;
+
+	snprintf(mz_name, sizeof(mz_name) - 1, "%lx",
+		 (unsigned long)rte_get_timer_cycles());
+	if (core_id == (unsigned int)LCORE_ID_ANY)
+		core_id = rte_get_master_lcore();
+	socket_id = rte_lcore_to_socket_id(core_id);
+	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
+	if (!mz) {
+		rte_panic("Unable to allocate DMA memory of size %zu bytes - %s\n",
+			  size, rte_strerror(rte_errno));
+		*phys = 0;
+		return NULL;
+	}
+	*phys = mz->iova;
+	return mz->addr;
+}
+
+static int
+dlb_pf_ldb_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_ldb_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	struct dlb_port_memory port_memory;
+	int ret;
+	uint8_t *port_base;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = false;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&pc_dma_base,
+					       alloc_sz,
+					       PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0)
+		rte_panic("dlb pf pmd could not lock page for device i/o\n");
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_ldb_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response,
+				     NOT_VF_REQ,
+				     PF_ID_ZERO);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_LDB].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_LDB].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_LDB].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_LDB].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+
+	memset(&port_memory, 0, sizeof(port_memory));
+	dlb_list_init_head(&port_memory.list);
+
+	/* Fill out the per-port memory tracking structure */
+	dlb_dev->ldb_port_pages[response.id].pages = port_memory.pages;
+	dlb_dev->ldb_port_pages[response.id].domain_id = handle->domain_id;
+	dlb_dev->ldb_port_pages[response.id].cq_base =
+		(void *)(uintptr_t)cq_dma_base;
+	dlb_dev->ldb_port_pages[response.id].pc_base =
+		(void *)(uintptr_t)pc_dma_base;
+	dlb_dev->ldb_port_pages[response.id].cq_dma_base = cq_dma_base;
+	dlb_dev->ldb_port_pages[response.id].pc_dma_base = pc_dma_base;
+	dlb_dev->ldb_port_pages[response.id].valid = true;
+	dlb_list_splice(&port_memory.list,
+			&dlb_dev->ldb_port_pages[response.id].list);
+
+	ret = dlb_hw_enable_ldb_cq_interrupts(dlb_dev,
+					      response.id,
+					      cfg->cq_depth_threshold);
+	if (ret) /* Internal error, don't unwind port creation */
+		goto create_port_err;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	return ret;
+}
+
+static int
+dlb_pf_dir_port_create(struct dlb_hw_dev *handle,
+		       struct dlb_create_dir_port_args *cfg,
+		       enum dlb_cq_poll_modes poll_mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	struct dlb_port_memory port_memory;
+	int ret;
+	uint8_t *port_base;
+	int alloc_sz, qe_sz;
+	rte_iova_t pp_dma_base;
+	rte_iova_t pc_dma_base;
+	rte_iova_t cq_dma_base;
+	int is_dir = true;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	if (poll_mode == DLB_CQ_POLL_MODE_STD)
+		qe_sz = sizeof(struct dlb_dequeue_qe);
+	else
+		qe_sz = RTE_CACHE_LINE_SIZE;
+
+	/* Calculate the port memory required, including two cache lines for
+	 * credit pop counts. Round up to the nearest cache line.
+	 */
+	alloc_sz = 2 * RTE_CACHE_LINE_SIZE + cfg->cq_depth * qe_sz;
+	alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+	port_base = dlb_alloc_coherent_aligned(&pc_dma_base,
+					       alloc_sz,
+					       PAGE_SIZE);
+	if (port_base == NULL)
+		return -ENOMEM;
+
+	/* Lock the page in memory */
+	ret = rte_mem_lock_page(port_base);
+	if (ret < 0)
+		rte_panic("dlb pf pmd could not lock page for device i/o\n");
+
+	memset(port_base, 0, alloc_sz);
+	cq_dma_base = (uintptr_t)(pc_dma_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	ret = dlb_hw_create_dir_port(&dlb_dev->hw,
+				     handle->domain_id,
+				     cfg,
+				     pc_dma_base,
+				     cq_dma_base,
+				     &response,
+				     NOT_VF_REQ,
+				     PF_ID_ZERO);
+	if (ret)
+		goto create_port_err;
+
+	pp_dma_base = (uintptr_t)dlb_dev->hw.func_kva + PP_BASE(is_dir);
+	dlb_port[response.id][DLB_DIR].pp_addr =
+		(void *)(uintptr_t)(pp_dma_base + (PAGE_SIZE * response.id));
+
+	dlb_port[response.id][DLB_DIR].cq_base =
+		(void *)(uintptr_t)(port_base + (2 * RTE_CACHE_LINE_SIZE));
+
+	dlb_port[response.id][DLB_DIR].ldb_popcount =
+		(void *)(uintptr_t)port_base;
+	dlb_port[response.id][DLB_DIR].dir_popcount = (void *)(uintptr_t)
+		(port_base + RTE_CACHE_LINE_SIZE);
+
+	memset(&port_memory, 0, sizeof(port_memory));
+	dlb_list_init_head(&port_memory.list);
+
+	/* Fill out the per-port memory tracking structure */
+	dlb_dev->dir_port_pages[response.id].pages = port_memory.pages;
+	dlb_dev->dir_port_pages[response.id].domain_id = handle->domain_id;
+	dlb_dev->dir_port_pages[response.id].cq_base =
+		(void *)(uintptr_t)cq_dma_base;
+	dlb_dev->dir_port_pages[response.id].pc_base =
+		(void *)(uintptr_t)pc_dma_base;
+	dlb_dev->dir_port_pages[response.id].cq_dma_base = cq_dma_base;
+	dlb_dev->dir_port_pages[response.id].pc_dma_base = pc_dma_base;
+	dlb_dev->dir_port_pages[response.id].valid = true;
+	dlb_list_splice(&port_memory.list,
+			&dlb_dev->dir_port_pages[response.id].list);
+
+	ret = dlb_hw_enable_dir_cq_interrupts(dlb_dev,
+					      response.id,
+					      cfg->cq_depth_threshold);
+	if (ret) /* Internal error, don't unwind port creation */
+		goto create_port_err;
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+create_port_err:
+
+	return ret;
+}
+
+static int
+dlb_pf_map_qid(struct dlb_hw_dev *handle,
+	       struct dlb_map_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_map_qid(&dlb_dev->hw,
+			     handle->domain_id,
+			     cfg,
+			     &response,
+			     NOT_VF_REQ,
+			     PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_unmap_qid(struct dlb_hw_dev *handle,
+		 struct dlb_unmap_qid_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_unmap_qid(&dlb_dev->hw,
+			       handle->domain_id,
+			       cfg,
+			       &response,
+			       NOT_VF_REQ,
+			       PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response,
+				  NOT_VF_REQ,
+				  PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_block_on_cq_interrupt(struct dlb_hw_dev *handle,
+			     int port_id, bool is_ldb,
+			     volatile void *cq_va, uint8_t cq_gen,
+				       bool arm)
+{
+	RTE_SET_USED(handle);
+	RTE_SET_USED(port_id);
+	RTE_SET_USED(is_ldb);
+	RTE_SET_USED(cq_va);
+	RTE_SET_USED(cq_gen);
+	RTE_SET_USED(arm);
+
+	return 0;
+}
+
+static int
+dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
+			   struct dlb_pending_port_unmaps_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_pending_port_unmaps(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response,
+					 NOT_VF_REQ,
+					 PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_ldb_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_ldb_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_ldb_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response,
+					 NOT_VF_REQ,
+					 PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_dir_queue_depth(struct dlb_hw_dev *handle,
+			   struct dlb_get_dir_queue_depth_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret = 0;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_get_dir_queue_depth(&dlb_dev->hw,
+					 handle->domain_id,
+					 args,
+					 &response,
+					 NOT_VF_REQ,
+					 PF_ID_ZERO);
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
+dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
+			enum dlb_cq_poll_modes *mode)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+
+	if (dlb_dev->revision >= DLB_REV_B0)
+		*mode = DLB_CQ_POLL_MODE_SPARSE;
+	else
+		*mode = DLB_CQ_POLL_MODE_STD;
+
+	return 0;
+}
+
+static int
+dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_get_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
+			 struct dlb_set_sn_allocation_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
+					     args->num);
+
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static int
+dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
+			struct dlb_get_sn_occupancy_args *args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
+						      args->group);
+
+	response.id = ret;
+	response.status = 0;
+
+	*(struct dlb_cmd_response *)args->response = response;
+
+	return ret;
+}
+
+static void
+dlb_pf_iface_fn_ptrs_init(void)
+{
+	dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
+	dlb_iface_open = dlb_pf_open;
+	dlb_iface_domain_close = dlb_pf_domain_close;
+	dlb_iface_get_driver_version = NULL; /*dlb_pf_get_driver_version;*/
+	dlb_iface_get_device_version = dlb_pf_get_device_version;
+	dlb_iface_get_num_resources = dlb_pf_get_num_resources;
+	dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
+	dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
+	dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
+	dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
+	dlb_iface_dir_queue_create = dlb_pf_dir_queue_create;
+	dlb_iface_ldb_port_create = dlb_pf_ldb_port_create;
+	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
+	dlb_iface_map_qid = dlb_pf_map_qid;
+	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
+	dlb_iface_block_on_cq_interrupt = dlb_pf_block_on_cq_interrupt;
+	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
+	dlb_iface_get_ldb_queue_depth = dlb_pf_get_ldb_queue_depth;
+	dlb_iface_get_dir_queue_depth = dlb_pf_get_dir_queue_depth;
+	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
+	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
+	dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
+	dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
+}
+
+/* PCI DEV HOOKS */
+static int
+dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
+{
+	int ret = 0;
+	struct rte_pci_device *pci_dev;
+	struct dlb_devargs dlb_args = {
+		.socket_id = rte_socket_id(),
+		.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+		.num_dir_credits_override = -1,
+		.defer_sched = 0,
+		.num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
+	};
+	struct dlb_eventdev *dlb;
+
+	DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
+		    eventdev->data->dev_id, eventdev->data->socket_id);
+
+	dlb_entry_points_init(eventdev);
+
+	dlb_pf_iface_fn_ptrs_init();
+
+	pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
+
+		/* Probe the DLB PF layer */
+		dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
+
+		if (dlb->qm_instance.pf_dev == NULL) {
+			DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
+				    rte_errno);
+			ret = -rte_errno;
+			goto dlb_probe_failed;
+		}
+
+		/* Were we invoked with runtime parameters? */
+		if (pci_dev->device.devargs) {
+			ret = dlb_parse_params(pci_dev->device.devargs->args,
+					       pci_dev->device.devargs->name,
+					       &dlb_args);
+			if (ret) {
+				DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
+					    ret, rte_errno);
+				goto dlb_probe_failed;
+			}
+		}
+
+		ret = dlb_primary_eventdev_probe(eventdev,
+						 event_dlb_pf_name,
+						 &dlb_args,
+						 DLB_NOT_VDEV);
+	} else {
+		ret = dlb_secondary_eventdev_probe(eventdev,
+						   event_dlb_pf_name,
+						   DLB_NOT_VDEV);
+	}
+	if (ret)
+		goto dlb_probe_failed;
+
+	DLB_LOG_INFO("DLB PF Probe success\n");
+
+	return 0;
+
+dlb_probe_failed:
+
+	DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
+
+	return ret;
+}
+
+#define EVENTDEV_INTEL_VENDOR_ID 0x8086
+
+static const struct rte_pci_id pci_id_dlb_map[] = {
+	{
+		RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
+			       DLB_PF_DEV_ID)
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static int
+event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
+		    struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
+		sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
+		event_dlb_pf_name);
+}
+
+static int
+event_dlb_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_event_pmd_pci_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver pci_eventdev_dlb_pmd = {
+	.id_table = pci_id_dlb_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+	.probe = event_dlb_pci_probe,
+	.remove = event_dlb_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);
-- 
2.13.6

[dpdk-dev] [PATCH 11/27] event/dlb: add flexible PMD to device interfaces

[McDaniel, Timothy]

The PMD uses function pointers so that it can be executed as a bifurcated
PMD accessing hardware via a linux kernel mode driver, or as a PF PMD
where it has complete control of the hardware device.

Note that this interface is not used in the data path.

Change-Id: I42a3f9d7370e9a7d89855ea1327cb31df4eebf85
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>

Change-Id: I3cce87a28b95cf6e528420da89c8435f41a233ee
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_iface.c | 105 ++++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h |  92 ++++++++++++++++++++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h

diff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c
new file mode 100644
index 000000000..72f245220
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.c
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <rte_debug.h>
+#include <rte_bus_pci.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_get_driver_version)(struct dlb_hw_dev *handle,
+				    struct dlb_cmd_response *response);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
+int (*dlb_iface_block_on_cq_interrupt)(struct dlb_hw_dev *handle,
+				       int port_id, bool is_ldb,
+				       volatile void *cq_va, uint8_t cq_gen,
+				       bool arm);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
+int (*dlb_iface_enqueue_domain_alert)(struct dlb_hw_dev *handle,
+				      uint64_t alert_data);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
+void (*dlb_iface_port_mmap)(struct dlb_port *qm_port);
diff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h
new file mode 100644
index 000000000..8c3d88bfe
--- /dev/null
+++ b/drivers/event/dlb/dlb_iface.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_get_driver_version)(struct dlb_hw_dev *handle,
+				    struct dlb_cmd_response *response);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
+extern int (*dlb_iface_block_on_cq_interrupt)(struct dlb_hw_dev *handle,
+				       int port_id, bool is_ldb,
+				       volatile void *cq_va, uint8_t cq_gen,
+				       bool arm);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
+extern int (*dlb_iface_enqueue_domain_alert)(struct dlb_hw_dev *handle,
+				      uint64_t alert_data);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
+extern void (*dlb_iface_port_mmap)(struct dlb_port *qm_port);
+
+#endif /* _DLB_IFACE_H */
-- 
2.13.6

[dpdk-dev] [PATCH 12/27] event/dlb: add the PMD's public interfaces

[McDaniel, Timothy]

Change-Id: I0237d00b8d27d84962be467c48de7c4d5137cc4c
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/rte_pmd_dlb.c                 | 39 ++++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h                 | 69 +++++++++++++++++++++++++
 drivers/event/dlb/rte_pmd_dlb_event_version.map |  6 +++
 3 files changed, 114 insertions(+)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/rte_pmd_dlb_event_version.map

diff --git a/drivers/event/dlb/rte_pmd_dlb.c b/drivers/event/dlb/rte_pmd_dlb.c
new file mode 100644
index 000000000..11f0d66cd
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.c
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+				uint8_t port_id,
+				enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
diff --git a/drivers/event/dlb/rte_pmd_dlb.h b/drivers/event/dlb/rte_pmd_dlb.h
new file mode 100644
index 000000000..1b637def6
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2020 Intel Corporation
+ */
+
+/*!
+ *  @file      rte_pmd_dlb.h
+ *
+ *  @brief     DLB PMD-specific functions
+ */
+
+#ifndef _RTE_PMD_DLB_H_
+#define _RTE_PMD_DLB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/**
+ * Selects the token pop mode for an DLB port.
+ */
+enum dlb_token_pop_mode {
+	/* Pop the CQ tokens immediately after dequeueing. */
+	AUTO_POP,
+	/* Pop CQ tokens after (dequeue_depth - 1) events are released.
+	 * Supported on load-balanced ports only.
+	 */
+	DELAYED_POP,
+	/* Pop the CQ tokens during next dequeue operation. */
+	DEFERRED_POP,
+
+	/* NUM_TOKEN_POP_MODES must be last */
+	NUM_TOKEN_POP_MODES
+};
+
+/*!
+ * Configure the token pop mode for an DLB port. By default, all ports use
+ * AUTO_POP. This function must be called before calling rte_event_port_setup()
+ * for the port, but after calling rte_event_dev_configure().
+ *
+ * @note
+ *    The defer_sched vdev arg, which configures all load-balanced ports with
+ *    dequeue_depth == 1 for DEFERRED_POP mode, takes precedence over this
+ *    function.
+ *
+ * @param dev_id
+ *    The identifier of the event device.
+ * @param port_id
+ *    The identifier of the event port.
+ * @param mode
+ *    The token pop mode.
+ *
+ * @return
+ * - 0: Success
+ * - EINVAL: Invalid dev_id, port_id, or mode
+ * - EINVAL: The DLB is not configured, is already running, or the port is
+ *   already setup
+ */
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+				uint8_t port_id,
+				enum dlb_token_pop_mode mode);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_DLB_H_ */
diff --git a/drivers/event/dlb/rte_pmd_dlb_event_version.map b/drivers/event/dlb/rte_pmd_dlb_event_version.map
new file mode 100644
index 000000000..8b1c796de
--- /dev/null
+++ b/drivers/event/dlb/rte_pmd_dlb_event_version.map
@@ -0,0 +1,6 @@
+DPDK_20.08 {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+	local: *;
+};
-- 
2.13.6

[dpdk-dev] [PATCH 13/27] event/dlb: add xstats support

[McDaniel, Timothy]

Change-Id: Ia7a000e12fcd035f66280fbbd4f17b4ccc95cc2c
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_xstats.c | 1251 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1251 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c

diff --git a/drivers/event/dlb/dlb_xstats.c b/drivers/event/dlb/dlb_xstats.c
new file mode 100644
index 000000000..1b361ad20
--- /dev/null
+++ b/drivers/event/dlb/dlb_xstats.c
@@ -0,0 +1,1251 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include <rte_debug.h>
+#include <rte_bus_vdev.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_vdev.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+enum dlb_xstats_type {
+	/* common to device and port */
+	rx_ok,				/**< Receive an event */
+	rx_drop,                        /**< Error bit set in received QE */
+	rx_interrupt_wait,		/**< Wait on an interrupt */
+	rx_umonitor_umwait,		/**< Block using umwait */
+	tx_ok,				/**< Transmit an event */
+	total_polls,			/**< Call dequeue_burst */
+	zero_polls,			/**< Call dequeue burst and return 0 */
+	tx_nospc_ldb_hw_credits,	/**< Insufficient LDB h/w credits */
+	tx_nospc_dir_hw_credits,	/**< Insufficient DIR h/w credits */
+	tx_nospc_inflight_max,		/**< Reach the new_event_threshold */
+	tx_nospc_new_event_limit,	/**< Insufficient s/w credits */
+	tx_nospc_inflight_credits,	/**< Port has too few s/w credits */
+	/* device specific */
+	nb_events_limit,
+	inflight_events,
+	ldb_pool_size,
+	dir_pool_size,
+	/* port specific */
+	tx_new,				/**< Send an OP_NEW event */
+	tx_fwd,				/**< Send an OP_FORWARD event */
+	tx_rel,				/**< Send an OP_RELEASE event */
+	tx_implicit_rel,		/**< Issue an implicit event release */
+	tx_sched_ordered,		/**< Send a SCHED_TYPE_ORDERED event */
+	tx_sched_unordered,		/**< Send a SCHED_TYPE_PARALLEL event */
+	tx_sched_atomic,		/**< Send a SCHED_TYPE_ATOMIC event */
+	tx_sched_directed,		/**< Send a directed event */
+	tx_invalid,                     /**< Send an event with an invalid op */
+	outstanding_releases,		/**< # of releases a port owes */
+	max_outstanding_releases,	/**< max # of releases a port can owe */
+	rx_sched_ordered,		/**< Dequeue an ordered event */
+	rx_sched_unordered,		/**< Dequeue an unordered event */
+	rx_sched_atomic,		/**< Dequeue an atomic event */
+	rx_sched_directed,		/**< Dequeue an directed event */
+	rx_sched_invalid,               /**< Dequeue event sched type invalid */
+	/* common to port and queue */
+	is_configured,			/**< Port is configured */
+	is_load_balanced,		/**< Port is LDB */
+	hw_id,				/**< Hardware ID */
+	/* queue specific */
+	num_links,			/**< Number of ports linked */
+	sched_type,			/**< Queue sched type */
+	enq_ok,				/**< # events enqueued to the queue */
+	current_depth			/**< Current queue depth */
+};
+
+typedef uint64_t (*dlb_xstats_fn)(struct dlb_eventdev *dlb,
+		uint16_t obj_idx, /* port or queue id */
+		enum dlb_xstats_type stat, int extra_arg);
+
+enum dlb_xstats_fn_type {
+	DLB_XSTATS_FN_DEV,
+	DLB_XSTATS_FN_PORT,
+	DLB_XSTATS_FN_QUEUE
+};
+
+struct dlb_xstats_entry {
+	struct rte_event_dev_xstats_name name;
+	uint64_t reset_value; /* an offset to be taken away to emulate resets */
+	enum dlb_xstats_fn_type fn_id;
+	enum dlb_xstats_type stat;
+	enum rte_event_dev_xstats_mode mode;
+	int extra_arg;
+	uint16_t obj_idx;
+	uint8_t reset_allowed; /* when set, this value can be reset */
+};
+
+/* Some device stats are simply a summation of the corresponding port values */
+static uint64_t
+dlb_device_traffic_stat_get(struct dlb_eventdev *dlb, int which_stat)
+{
+	int i;
+	uint64_t val = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+		struct dlb_eventdev_port *port = &dlb->ev_ports[i];
+
+		if (!port->setup_done)
+			continue;
+
+		switch (which_stat) {
+		case rx_ok:
+			val += port->stats.traffic.rx_ok;
+			break;
+		case rx_drop:
+			val += port->stats.traffic.rx_drop;
+			break;
+		case rx_interrupt_wait:
+			val += port->stats.traffic.rx_interrupt_wait;
+			break;
+		case rx_umonitor_umwait:
+			val += port->stats.traffic.rx_umonitor_umwait;
+			break;
+		case tx_ok:
+			val += port->stats.traffic.tx_ok;
+			break;
+		case total_polls:
+			val += port->stats.traffic.total_polls;
+			break;
+		case zero_polls:
+			val += port->stats.traffic.zero_polls;
+			break;
+		case tx_nospc_ldb_hw_credits:
+			val += port->stats.traffic.tx_nospc_ldb_hw_credits;
+			break;
+		case tx_nospc_dir_hw_credits:
+			val += port->stats.traffic.tx_nospc_dir_hw_credits;
+			break;
+		case tx_nospc_inflight_max:
+			val += port->stats.traffic.tx_nospc_inflight_max;
+			break;
+		case tx_nospc_new_event_limit:
+			val += port->stats.traffic.tx_nospc_new_event_limit;
+			break;
+		case tx_nospc_inflight_credits:
+			val += port->stats.traffic.tx_nospc_inflight_credits;
+			break;
+		default:
+			return -1;
+		}
+	}
+	return val;
+}
+
+static uint64_t
+get_dev_stat(struct dlb_eventdev *dlb, uint16_t obj_idx __rte_unused,
+	     enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	switch (type) {
+	case rx_ok:
+	case rx_drop:
+	case rx_interrupt_wait:
+	case rx_umonitor_umwait:
+	case tx_ok:
+	case total_polls:
+	case zero_polls:
+	case tx_nospc_ldb_hw_credits:
+	case tx_nospc_dir_hw_credits:
+	case tx_nospc_inflight_max:
+	case tx_nospc_new_event_limit:
+	case tx_nospc_inflight_credits:
+		return dlb_device_traffic_stat_get(dlb, type);
+	case nb_events_limit:
+		return dlb->new_event_limit;
+	case inflight_events:
+		return rte_atomic32_read(&dlb->inflights);
+	case ldb_pool_size:
+		return dlb->num_ldb_credits;
+	case dir_pool_size:
+		return dlb->num_dir_credits;
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_port_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	      enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_port *ev_port = &dlb->ev_ports[obj_idx];
+
+	switch (type) {
+	case rx_ok: return ev_port->stats.traffic.rx_ok;
+
+	case rx_drop: return ev_port->stats.traffic.rx_drop;
+
+	case rx_interrupt_wait: return ev_port->stats.traffic.rx_interrupt_wait;
+
+	case rx_umonitor_umwait:
+		return ev_port->stats.traffic.rx_umonitor_umwait;
+
+	case tx_ok: return ev_port->stats.traffic.tx_ok;
+
+	case total_polls: return ev_port->stats.traffic.total_polls;
+
+	case zero_polls: return ev_port->stats.traffic.zero_polls;
+
+	case tx_nospc_ldb_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_ldb_hw_credits;
+
+	case tx_nospc_dir_hw_credits:
+		return ev_port->stats.traffic.tx_nospc_dir_hw_credits;
+
+	case tx_nospc_inflight_max:
+		return ev_port->stats.traffic.tx_nospc_inflight_max;
+
+	case tx_nospc_new_event_limit:
+		return ev_port->stats.traffic.tx_nospc_new_event_limit;
+
+	case tx_nospc_inflight_credits:
+		return ev_port->stats.traffic.tx_nospc_inflight_credits;
+
+	case is_configured: return ev_port->setup_done;
+
+	case is_load_balanced: return !ev_port->qm_port.is_directed;
+
+	case hw_id: return ev_port->qm_port.id;
+
+	case tx_new: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_NEW];
+
+	case tx_fwd: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD];
+
+	case tx_rel: return ev_port->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE];
+
+	case tx_implicit_rel: return ev_port->stats.tx_implicit_rel;
+
+	case tx_sched_ordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case tx_sched_unordered:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case tx_sched_atomic:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case tx_sched_directed:
+		return ev_port->stats.tx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case tx_invalid: return ev_port->stats.tx_invalid;
+
+	case outstanding_releases: return ev_port->outstanding_releases;
+
+	case max_outstanding_releases:
+		return DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	case rx_sched_ordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ORDERED];
+
+	case rx_sched_unordered:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_UNORDERED];
+
+	case rx_sched_atomic:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_ATOMIC];
+
+	case rx_sched_directed:
+		return ev_port->stats.rx_sched_cnt[DLB_SCHED_DIRECTED];
+
+	case rx_sched_invalid: return ev_port->stats.rx_sched_invalid;
+
+	default: return -1;
+	}
+}
+
+static uint64_t
+get_queue_stat(struct dlb_eventdev *dlb, uint16_t obj_idx,
+	       enum dlb_xstats_type type, int extra_arg __rte_unused)
+{
+	struct dlb_eventdev_queue *ev_queue =
+		&dlb->ev_queues[obj_idx];
+
+	switch (type) {
+	case is_configured: return ev_queue->setup_done;
+
+	case is_load_balanced: return !ev_queue->qm_queue.is_directed;
+
+	case hw_id: return ev_queue->qm_queue.id;
+
+	case num_links: return ev_queue->num_links;
+
+	case sched_type: return ev_queue->qm_queue.sched_type;
+
+	case enq_ok:
+	{
+		int port_count = 0;
+		uint64_t enq_ok_tally = 0;
+
+		ev_queue->enq_ok = 0;
+		for (port_count = 0; port_count < DLB_MAX_NUM_PORTS;
+		     port_count++) {
+			struct dlb_eventdev_port *ev_port =
+				&dlb->ev_ports[port_count];
+			enq_ok_tally += ev_port->stats.enq_ok[ev_queue->id];
+		}
+		ev_queue->enq_ok = enq_ok_tally;
+		return ev_queue->enq_ok;
+	}
+
+	case current_depth: return dlb_get_queue_depth(dlb, ev_queue);
+
+	default: return -1;
+	}
+}
+
+int
+dlb_xstats_init(struct dlb_eventdev *dlb)
+{
+	/*
+	 * define the stats names and types. Used to build up the device
+	 * xstats array
+	 * There are multiple set of stats:
+	 *   - device-level,
+	 *   - per-port,
+	 *   - per-qid,
+	 *
+	 * For each of these sets, we have three parallel arrays, one for the
+	 * names, the other for the stat type parameter to be passed in the fn
+	 * call to get that stat. The third array allows resetting or not.
+	 * All these arrays must be kept in sync
+	 */
+	static const char * const dev_stats[] = {
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"nb_events_limit",
+		"inflight_events",
+		"ldb_pool_size",
+		"dir_pool_size",
+	};
+	static const enum dlb_xstats_type dev_types[] = {
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		nb_events_limit,
+		inflight_events,
+		ldb_pool_size,
+		dir_pool_size,
+	};
+	/* Note: generated device stats are not allowed to be reset. */
+	static const uint8_t dev_reset_allowed[] = {
+		0, /* rx_ok */
+		0, /* rx_drop */
+		0, /* rx_interrupt_wait */
+		0, /* rx_umonitor_umwait */
+		0, /* tx_ok */
+		0, /* total_polls */
+		0, /* zero_polls */
+		0, /* tx_nospc_ldb_hw_credits */
+		0, /* tx_nospc_dir_hw_credits */
+		0, /* tx_nospc_inflight_max */
+		0, /* tx_nospc_new_event_limit */
+		0, /* tx_nospc_inflight_credits */
+		0, /* nb_events_limit */
+		0, /* inflight_events */
+		0, /* ldb_pool_size */
+		0, /* dir_pool_size */
+	};
+	static const char * const port_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"rx_ok",
+		"rx_drop",
+		"rx_interrupt_wait",
+		"rx_umonitor_umwait",
+		"tx_ok",
+		"total_polls",
+		"zero_polls",
+		"tx_nospc_ldb_hw_credits",
+		"tx_nospc_dir_hw_credits",
+		"tx_nospc_inflight_max",
+		"tx_nospc_new_event_limit",
+		"tx_nospc_inflight_credits",
+		"tx_new",
+		"tx_fwd",
+		"tx_rel",
+		"tx_implicit_rel",
+		"tx_sched_ordered",
+		"tx_sched_unordered",
+		"tx_sched_atomic",
+		"tx_sched_directed",
+		"tx_invalid",
+		"outstanding_releases",
+		"max_outstanding_releases",
+		"rx_sched_ordered",
+		"rx_sched_unordered",
+		"rx_sched_atomic",
+		"rx_sched_directed",
+		"rx_sched_invalid"
+	};
+	static const enum dlb_xstats_type port_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		rx_ok,
+		rx_drop,
+		rx_interrupt_wait,
+		rx_umonitor_umwait,
+		tx_ok,
+		total_polls,
+		zero_polls,
+		tx_nospc_ldb_hw_credits,
+		tx_nospc_dir_hw_credits,
+		tx_nospc_inflight_max,
+		tx_nospc_new_event_limit,
+		tx_nospc_inflight_credits,
+		tx_new,
+		tx_fwd,
+		tx_rel,
+		tx_implicit_rel,
+		tx_sched_ordered,
+		tx_sched_unordered,
+		tx_sched_atomic,
+		tx_sched_directed,
+		tx_invalid,
+		outstanding_releases,
+		max_outstanding_releases,
+		rx_sched_ordered,
+		rx_sched_unordered,
+		rx_sched_atomic,
+		rx_sched_directed,
+		rx_sched_invalid
+	};
+	static const uint8_t port_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		1, /* rx_ok */
+		1, /* rx_drop */
+		1, /* rx_interrupt_wait */
+		1, /* rx_umonitor_umwait */
+		1, /* tx_ok */
+		1, /* total_polls */
+		1, /* zero_polls */
+		1, /* tx_nospc_ldb_hw_credits */
+		1, /* tx_nospc_dir_hw_credits */
+		1, /* tx_nospc_inflight_max */
+		1, /* tx_nospc_new_event_limit */
+		1, /* tx_nospc_inflight_credits */
+		1, /* tx_new */
+		1, /* tx_fwd */
+		1, /* tx_rel */
+		1, /* tx_implicit_rel */
+		1, /* tx_sched_ordered */
+		1, /* tx_sched_unordered */
+		1, /* tx_sched_atomic */
+		1, /* tx_sched_directed */
+		1, /* tx_invalid */
+		0, /* outstanding_releases */
+		0, /* max_outstanding_releases */
+		1, /* rx_sched_ordered */
+		1, /* rx_sched_unordered */
+		1, /* rx_sched_atomic */
+		1, /* rx_sched_directed */
+		1  /* rx_sched_invalid */
+	};
+
+	/* QID specific stats */
+	static const char * const qid_stats[] = {
+		"is_configured",
+		"is_load_balanced",
+		"hw_id",
+		"num_links",
+		"sched_type",
+		"enq_ok",
+		"current_depth",
+	};
+	static const enum dlb_xstats_type qid_types[] = {
+		is_configured,
+		is_load_balanced,
+		hw_id,
+		num_links,
+		sched_type,
+		enq_ok,
+		current_depth,
+	};
+	static const uint8_t qid_reset_allowed[] = {
+		0, /* is_configured */
+		0, /* is_load_balanced */
+		0, /* hw_id */
+		0, /* num_links */
+		0, /* sched_type */
+		1, /* enq_ok */
+		0, /* current_depth */
+	};
+
+	/* ---- end of stat definitions ---- */
+
+	/* check sizes, since a missed comma can lead to strings being
+	 * joined by the compiler.
+	 */
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_types));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_types));
+
+	RTE_BUILD_BUG_ON(RTE_DIM(dev_stats) != RTE_DIM(dev_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(port_stats) != RTE_DIM(port_reset_allowed));
+	RTE_BUILD_BUG_ON(RTE_DIM(qid_stats) != RTE_DIM(qid_reset_allowed));
+
+	/* other vars */
+	const unsigned int count = RTE_DIM(dev_stats) +
+			DLB_MAX_NUM_PORTS * RTE_DIM(port_stats) +
+			DLB_MAX_NUM_QUEUES * RTE_DIM(qid_stats);
+	unsigned int i, port, qid, stat_id = 0;
+
+	dlb->xstats = rte_zmalloc_socket(NULL,
+					 sizeof(dlb->xstats[0]) * count, 0,
+					 dlb->qm_instance.info.socket_id);
+	if (dlb->xstats == NULL)
+		return -ENOMEM;
+
+#define sname dlb->xstats[stat_id].name.name
+	for (i = 0; i < RTE_DIM(dev_stats); i++, stat_id++) {
+		dlb->xstats[stat_id] = (struct dlb_xstats_entry) {
+			.fn_id = DLB_XSTATS_FN_DEV,
+			.stat = dev_types[i],
+			.mode = RTE_EVENT_DEV_XSTATS_DEVICE,
+			.reset_allowed = dev_reset_allowed[i],
+		};
+		snprintf(sname, sizeof(sname), "dev_%s", dev_stats[i]);
+	}
+	dlb->xstats_count_mode_dev = stat_id;
+
+	for (port = 0; port < DLB_MAX_NUM_PORTS; port++) {
+		dlb->xstats_offset_for_port[port] = stat_id;
+
+		uint32_t count_offset = stat_id;
+
+		for (i = 0; i < RTE_DIM(port_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_PORT,
+				.obj_idx = port,
+				.stat = port_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_PORT,
+				.reset_allowed = port_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "port_%u_%s",
+				 port, port_stats[i]);
+		}
+
+		dlb->xstats_count_per_port[port] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_port = stat_id - dlb->xstats_count_mode_dev;
+
+	for (qid = 0; qid < DLB_MAX_NUM_QUEUES; qid++) {
+		uint32_t count_offset = stat_id;
+
+		dlb->xstats_offset_for_qid[qid] = stat_id;
+
+		for (i = 0; i < RTE_DIM(qid_stats); i++, stat_id++) {
+			dlb->xstats[stat_id] = (struct dlb_xstats_entry){
+				.fn_id = DLB_XSTATS_FN_QUEUE,
+				.obj_idx = qid,
+				.stat = qid_types[i],
+				.mode = RTE_EVENT_DEV_XSTATS_QUEUE,
+				.reset_allowed = qid_reset_allowed[i],
+			};
+			snprintf(sname, sizeof(sname), "qid_%u_%s",
+				 qid, qid_stats[i]);
+		}
+
+		dlb->xstats_count_per_qid[qid] = stat_id - count_offset;
+	}
+
+	dlb->xstats_count_mode_queue = stat_id -
+		(dlb->xstats_count_mode_dev + dlb->xstats_count_mode_port);
+#undef sname
+
+	dlb->xstats_count = stat_id;
+
+	return 0;
+}
+
+void
+dlb_xstats_uninit(struct dlb_eventdev *dlb)
+{
+	rte_free(dlb->xstats);
+	dlb->xstats_count = 0;
+}
+
+int
+dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		struct rte_event_dev_xstats_name *xstats_names,
+		unsigned int *ids, unsigned int size)
+{
+	const struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	unsigned int xidx = 0;
+
+	RTE_SET_USED(mode);
+	RTE_SET_USED(queue_port_id);
+
+	uint32_t xstats_mode_count = 0;
+	uint32_t start_offset = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			break;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		start_offset = dlb->xstats_offset_for_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			break;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		start_offset = dlb->xstats_offset_for_qid[queue_port_id];
+		break;
+	default:
+		return -EINVAL;
+	};
+
+	if (xstats_mode_count > size || !ids || !xstats_names)
+		return xstats_mode_count;
+
+	for (i = 0; i < dlb->xstats_count && xidx < size; i++) {
+		if (dlb->xstats[i].mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != dlb->xstats[i].obj_idx)
+			continue;
+
+		xstats_names[xidx] = dlb->xstats[i].name;
+		if (ids)
+			ids[xidx] = start_offset + xidx;
+		xidx++;
+	}
+	return xidx;
+}
+
+static int
+dlb_xstats_update(struct dlb_eventdev *dlb,
+		enum rte_event_dev_xstats_mode mode,
+		uint8_t queue_port_id, const unsigned int ids[],
+		uint64_t values[], unsigned int n, const uint32_t reset)
+{
+	unsigned int i;
+	unsigned int xidx = 0;
+
+	RTE_SET_USED(mode);
+	RTE_SET_USED(queue_port_id);
+
+	uint32_t xstats_mode_count = 0;
+
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		xstats_mode_count = dlb->xstats_count_mode_dev;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id >= DLB_MAX_NUM_PORTS)
+			goto invalid_value;
+		xstats_mode_count = dlb->xstats_count_per_port[queue_port_id];
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+#if (DLB_MAX_NUM_QUEUES <= 255) /* max 8 bit value */
+		if (queue_port_id >= DLB_MAX_NUM_QUEUES)
+			goto invalid_value;
+#endif
+		xstats_mode_count = dlb->xstats_count_per_qid[queue_port_id];
+		break;
+	default:
+		goto invalid_value;
+	};
+
+	for (i = 0; i < n && xidx < xstats_mode_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[ids[i]];
+		dlb_xstats_fn fn;
+
+		if (ids[i] > dlb->xstats_count || xs->mode != mode)
+			continue;
+
+		if (mode != RTE_EVENT_DEV_XSTATS_DEVICE &&
+		    queue_port_id != xs->obj_idx)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			rte_panic("Unexpected xstat fn_id %d\n", xs->fn_id);
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+
+		if (values)
+			values[xidx] = val;
+
+		if (xs->reset_allowed && reset)
+			xs->reset_value += val;
+
+		xidx++;
+	}
+
+	return xidx;
+
+invalid_value:
+	return -EINVAL;
+}
+
+int
+dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+		enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
+		const unsigned int ids[], uint64_t values[], unsigned int n)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	const uint32_t reset = 0;
+
+	return dlb_xstats_update(dlb, mode, queue_port_id, ids, values, n,
+				  reset);
+}
+
+uint64_t
+dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+				const char *name, unsigned int *id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	unsigned int i;
+	dlb_xstats_fn fn;
+
+	for (i = 0; i < dlb->xstats_count; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (strncmp(xs->name.name, name,
+			    RTE_EVENT_DEV_XSTATS_NAME_SIZE) == 0){
+			if (id != NULL)
+				*id = i;
+
+			switch (xs->fn_id) {
+			case DLB_XSTATS_FN_DEV:
+				fn = get_dev_stat;
+				break;
+			case DLB_XSTATS_FN_PORT:
+				fn = get_port_stat;
+				break;
+			case DLB_XSTATS_FN_QUEUE:
+				fn = get_queue_stat;
+				break;
+			default:
+				rte_panic("Unexpected xstat fn_id %d\n",
+					  xs->fn_id);
+			}
+
+			return fn(dlb, xs->obj_idx, xs->stat,
+				  xs->extra_arg) - xs->reset_value;
+		}
+	}
+	if (id != NULL)
+		*id = (uint32_t)-1;
+	return (uint64_t)-1;
+}
+
+static void
+dlb_xstats_reset_range(struct dlb_eventdev *dlb, uint32_t start,
+		       uint32_t num)
+{
+	uint32_t i;
+	dlb_xstats_fn fn;
+
+	for (i = start; i < start + num; i++) {
+		struct dlb_xstats_entry *xs = &dlb->xstats[i];
+
+		if (!xs->reset_allowed)
+			continue;
+
+		switch (xs->fn_id) {
+		case DLB_XSTATS_FN_DEV:
+			fn = get_dev_stat;
+			break;
+		case DLB_XSTATS_FN_PORT:
+			fn = get_port_stat;
+			break;
+		case DLB_XSTATS_FN_QUEUE:
+			fn = get_queue_stat;
+			break;
+		default:
+			rte_panic("Unexpected xstat fn_id %d\n", xs->fn_id);
+		}
+
+		uint64_t val = fn(dlb, xs->obj_idx, xs->stat, xs->extra_arg);
+		xs->reset_value = val;
+	}
+}
+
+static int
+dlb_xstats_reset_queue(struct dlb_eventdev *dlb, uint8_t queue_id,
+		       const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_QUEUE,
+					queue_id, ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	if (ids == NULL)
+		dlb_xstats_reset_range(dlb,
+				       dlb->xstats_offset_for_qid[queue_id],
+				       dlb->xstats_count_per_qid[queue_id]);
+
+	return 0;
+}
+
+static int
+dlb_xstats_reset_port(struct dlb_eventdev *dlb, uint8_t port_id,
+		      const uint32_t ids[], uint32_t nb_ids)
+{
+	const uint32_t reset = 1;
+	int offset = dlb->xstats_offset_for_port[port_id];
+	int nb_stat = dlb->xstats_count_per_port[port_id];
+
+	if (ids) {
+		uint32_t nb_reset = dlb_xstats_update(dlb,
+					RTE_EVENT_DEV_XSTATS_PORT, port_id,
+					ids, NULL, nb_ids,
+					reset);
+		return nb_reset == nb_ids ? 0 : -EINVAL;
+	}
+
+	dlb_xstats_reset_range(dlb, offset, nb_stat);
+	return 0;
+}
+
+static int
+dlb_xstats_reset_dev(struct dlb_eventdev *dlb, const uint32_t ids[],
+		     uint32_t nb_ids)
+{
+	uint32_t i;
+
+	if (ids) {
+		for (i = 0; i < nb_ids; i++) {
+			uint32_t id = ids[i];
+
+			if (id >= dlb->xstats_count_mode_dev)
+				return -EINVAL;
+			dlb_xstats_reset_range(dlb, id, 1);
+		}
+	} else {
+		for (i = 0; i < dlb->xstats_count_mode_dev; i++)
+			dlb_xstats_reset_range(dlb, i, 1);
+	}
+
+	return 0;
+}
+
+int
+dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			  enum rte_event_dev_xstats_mode mode,
+			  int16_t queue_port_id,
+			  const uint32_t ids[],
+			  uint32_t nb_ids)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	uint32_t i;
+
+	/* handle -1 for queue_port_id here, looping over all ports/queues */
+	switch (mode) {
+	case RTE_EVENT_DEV_XSTATS_DEVICE:
+		if (dlb_xstats_reset_dev(dlb, ids, nb_ids))
+			return -EINVAL;
+		break;
+	case RTE_EVENT_DEV_XSTATS_PORT:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
+				if (dlb_xstats_reset_port(dlb, i, ids,
+							  nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_PORTS) {
+			if (dlb_xstats_reset_port(dlb, queue_port_id, ids,
+						  nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	case RTE_EVENT_DEV_XSTATS_QUEUE:
+		if (queue_port_id == -1) {
+			for (i = 0; i < DLB_MAX_NUM_QUEUES; i++) {
+				if (dlb_xstats_reset_queue(dlb, i, ids,
+							   nb_ids))
+					return -EINVAL;
+			}
+		} else if (queue_port_id < DLB_MAX_NUM_QUEUES) {
+			if (dlb_xstats_reset_queue(dlb, queue_port_id, ids,
+						   nb_ids))
+				return -EINVAL;
+		} else {
+			return -EINVAL;
+		}
+		break;
+	};
+
+	return 0;
+}
+
+void
+dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	int i;
+
+	if (!f) {
+		printf("Invalid file pointer\n");
+		return;
+	}
+
+	if (!dev) {
+		fprintf(f, "Invalid event device\n");
+		return;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (!dlb) {
+		fprintf(f, "DLB Event device cannot be dumped!\n");
+		return;
+	}
+
+	if (!dlb->configured)
+		fprintf(f, "DLB Event device is not configured\n");
+
+	handle = &dlb->qm_instance;
+
+	fprintf(f, "================\n");
+	fprintf(f, "DLB Device Dump\n");
+	fprintf(f, "================\n");
+
+	fprintf(f, "Processor supports umonitor/umwait instructions = %s\n",
+		dlb->umwait_allowed ? "yes" : "no");
+
+	/* Generic top level device information */
+
+	fprintf(f, "device is configured and run state =");
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		fprintf(f, "STOPPED\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STOPPING)
+		fprintf(f, "STOPPING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTING)
+		fprintf(f, "STARTING\n");
+	else if (dlb->run_state == DLB_RUN_STATE_STARTED)
+		fprintf(f, "STARTED\n");
+	else
+		fprintf(f, "UNEXPECTED\n");
+
+	fprintf(f,
+		"dev ID=%d, dom ID=%u, name=%s, path=%s, sock=%u, evdev=%p\n",
+		handle->device_id, handle->domain_id, handle->device_name,
+		handle->device_path, handle->info.socket_id, dlb->event_dev);
+
+	fprintf(f, "domain device path=%s\n", handle->domain_device_path);
+
+	fprintf(f, "num dir ports=%u, num dir queues=%u\n",
+		dlb->num_dir_ports, dlb->num_dir_queues);
+
+	fprintf(f, "num ldb ports=%u, num ldb queues=%u\n",
+		dlb->num_ldb_ports, dlb->num_ldb_queues);
+
+	fprintf(f, "dir_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.dir_credit_pool_id, handle->cfg.num_dir_credits);
+
+	fprintf(f, "ldb_credit_pool_id=%u, num_credits=%u\n",
+		handle->cfg.ldb_credit_pool_id, handle->cfg.num_ldb_credits);
+
+	fprintf(f, "num atomic inflights=%u, hist list entries=%u\n",
+		handle->cfg.resources.num_atomic_inflights,
+		handle->cfg.resources.num_hist_list_entries);
+
+	fprintf(f, "results from most recent hw resource query:\n");
+
+	fprintf(f, "\tnum_sched_domains = %u\n",
+		dlb->hw_rsrc_query_results.num_sched_domains);
+
+	fprintf(f, "\tnum_ldb_queues = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_queues);
+
+	fprintf(f, "\tnum_ldb_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_ports);
+
+	fprintf(f, "\tnum_dir_ports = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_ports);
+
+	fprintf(f, "\tnum_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.num_atomic_inflights);
+
+	fprintf(f, "\tmax_contiguous_atomic_inflights = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_atomic_inflights);
+
+	fprintf(f, "\tnum_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.num_hist_list_entries);
+
+	fprintf(f, "\tmax_contiguous_hist_list_entries = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_hist_list_entries);
+
+	fprintf(f, "\tnum_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credits);
+
+	fprintf(f, "\tmax_contiguous_ldb_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits);
+
+	fprintf(f, "\tnum_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credits);
+
+	fprintf(f, "\tmax_contiguous_dir_credits = %u\n",
+		dlb->hw_rsrc_query_results.max_contiguous_dir_credits);
+
+	fprintf(f, "\tnum_ldb_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_ldb_credit_pools);
+
+	fprintf(f, "\tnum_dir_credit_pools = %u\n",
+		dlb->hw_rsrc_query_results.num_dir_credit_pools);
+
+	/* Port level information */
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *p = &dlb->ev_ports[i];
+		int j;
+
+		if (!p->enq_configured)
+			fprintf(f, "Port_%d is not configured\n", i);
+
+		fprintf(f, "Port_%d\n", i);
+		fprintf(f, "=======\n");
+
+		fprintf(f, "\tevport_%u is configured, setup done=%d\n",
+			p->id, p->setup_done);
+
+		fprintf(f, "\tconfig state=%d, port state=%d\n",
+			p->qm_port.config_state, p->qm_port.state);
+
+		fprintf(f, "\tport is %s\n",
+			p->qm_port.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\toutstanding releases=%u\n",
+			p->outstanding_releases);
+
+		fprintf(f, "\tinflight max=%u, inflight credits=%u\n",
+			p->inflight_max, p->inflight_credits);
+
+		fprintf(f, "\tcredit update quanta=%u, implicit release =%u\n",
+			p->credit_update_quanta, p->implicit_release);
+
+		fprintf(f, "\tnum_links=%d, queues -> ", p->num_links);
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (p->link[j].valid)
+				fprintf(f, "id=%u prio=%u ",
+					p->link[j].queue_id,
+					p->link[j].priority);
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\thardware port id=%u\n", p->qm_port.id);
+
+		fprintf(f, "\tcached_ldb_credits=%u\n",
+			p->qm_port.cached_ldb_credits);
+
+		fprintf(f, "\tldb_pushcount_at_credit_expiry = %u\n",
+			p->qm_port.ldb_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tldb_credits = %u\n",
+			p->qm_port.ldb_credits);
+
+		fprintf(f, "\tcached_dir_credits = %u\n",
+			p->qm_port.cached_dir_credits);
+
+		fprintf(f, "\tdir_pushcount_at_credit_expiry=%u\n",
+			p->qm_port.dir_pushcount_at_credit_expiry);
+
+		fprintf(f, "\tdir_credits = %u\n",
+			p->qm_port.dir_credits);
+
+		fprintf(f, "\tgenbit=%d, cq_idx=%d, cq_depth=%d\n",
+			p->qm_port.gen_bit,
+			p->qm_port.cq_idx,
+			p->qm_port.cq_depth);
+
+		fprintf(f, "\tuse reserved token scheme=%d, cq_rsvd_token_deficit=%u\n",
+			p->qm_port.use_rsvd_token_scheme,
+			p->qm_port.cq_rsvd_token_deficit);
+
+		fprintf(f, "\tinterrupt armed=%d\n",
+			p->qm_port.int_armed);
+
+		fprintf(f, "\tPort statistics\n");
+
+		fprintf(f, "\t\trx_ok %" PRIu64 "\n",
+			p->stats.traffic.rx_ok);
+
+		fprintf(f, "\t\trx_drop %" PRIu64 "\n",
+			p->stats.traffic.rx_drop);
+
+		fprintf(f, "\t\trx_interrupt_wait %" PRIu64 "\n",
+			p->stats.traffic.rx_interrupt_wait);
+
+		fprintf(f, "\t\trx_umonitor_umwait %" PRIu64 "\n",
+			p->stats.traffic.rx_umonitor_umwait);
+
+		fprintf(f, "\t\ttx_ok %" PRIu64 "\n",
+			p->stats.traffic.tx_ok);
+
+		fprintf(f, "\t\ttotal_polls %" PRIu64 "\n",
+			p->stats.traffic.total_polls);
+
+		fprintf(f, "\t\tzero_polls %" PRIu64 "\n",
+			p->stats.traffic.zero_polls);
+
+		fprintf(f, "\t\ttx_nospc_ldb_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_ldb_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_dir_hw_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_dir_hw_credits);
+
+		fprintf(f, "\t\ttx_nospc_inflight_max %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_max);
+
+		fprintf(f, "\t\ttx_nospc_new_event_limit %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_new_event_limit);
+
+		fprintf(f, "\t\ttx_nospc_inflight_credits %" PRIu64 "\n",
+			p->stats.traffic.tx_nospc_inflight_credits);
+
+		fprintf(f, "\t\ttx_new %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_NEW]);
+
+		fprintf(f, "\t\ttx_fwd %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_FORWARD]);
+
+		fprintf(f, "\t\ttx_rel %" PRIu64 "\n",
+			p->stats.tx_op_cnt[RTE_EVENT_OP_RELEASE]);
+
+		fprintf(f, "\t\ttx_implicit_rel %" PRIu64 "\n",
+			p->stats.tx_implicit_rel);
+
+		fprintf(f, "\t\ttx_sched_ordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\ttx_sched_unordered %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\ttx_sched_atomic %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\ttx_sched_directed %" PRIu64 "\n",
+			p->stats.tx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\ttx_invalid %" PRIu64 "\n",
+			p->stats.tx_invalid);
+
+		fprintf(f, "\t\trx_sched_ordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ORDERED]);
+
+		fprintf(f, "\t\trx_sched_unordered %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_UNORDERED]);
+
+		fprintf(f, "\t\trx_sched_atomic %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_ATOMIC]);
+
+		fprintf(f, "\t\trx_sched_directed %" PRIu64 "\n",
+			p->stats.rx_sched_cnt[DLB_SCHED_DIRECTED]);
+
+		fprintf(f, "\t\trx_sched_invalid %" PRIu64 "\n",
+			p->stats.rx_sched_invalid);
+	}
+
+	/* Queue level information */
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *q = &dlb->ev_queues[i];
+		int j, k;
+
+		if (!q->setup_done)
+			fprintf(f, "Queue_%d is not configured\n", i);
+
+		fprintf(f, "Queue_%d\n", i);
+		fprintf(f, "========\n");
+
+		fprintf(f, "\tevqueue_%u is set up\n", q->id);
+
+		fprintf(f, "\tqueue is %s\n",
+			q->qm_queue.is_directed ? "directed" : "load balanced");
+
+		fprintf(f, "\tnum_links=%d, ports -> ", q->num_links);
+
+		for (j = 0; j < dlb->num_ports; j++) {
+			struct dlb_eventdev_port *p = &dlb->ev_ports[j];
+
+			for (k = 0; k < DLB_MAX_NUM_QIDS_PER_LDB_CQ; k++) {
+				if (p->link[k].valid &&
+				    p->link[k].queue_id == q->id)
+					fprintf(f, "id=%u prio=%u ",
+						p->id, p->link[k].priority);
+			}
+		}
+		fprintf(f, "\n");
+
+		fprintf(f, "\tcurrent depth: %u events\n",
+			dlb_get_queue_depth(dlb, q));
+
+		fprintf(f, "\tnum qid inflights=%u, sched_type=%d\n",
+			q->qm_queue.num_qid_inflights, q->qm_queue.sched_type);
+	}
+}
-- 
2.13.6

[dpdk-dev] [PATCH 14/27] event/dlb: add PMD self-tests

[McDaniel, Timothy]

Change-Id: I1d0bc52b02ad21966c64736bc78e403f381e98ec
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb_selftest.c | 1628 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 1628 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c

diff --git a/drivers/event/dlb/dlb_selftest.c b/drivers/event/dlb/dlb_selftest.c
new file mode 100644
index 000000000..fd5cd95ec
--- /dev/null
+++ b/drivers/event/dlb/dlb_selftest.c
@@ -0,0 +1,1628 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/queue.h>
+
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_debug.h>
+#include <rte_ethdev.h>
+#include <rte_cycles.h>
+#include <rte_eventdev.h>
+#include <rte_pause.h>
+#include <rte_bus_vdev.h>
+
+#include "dlb_priv.h"
+#include "rte_pmd_dlb.h"
+
+#define MAX_PORTS 32
+#define MAX_QIDS 32
+#define DEFAULT_NUM_SEQ_NUMS 32
+
+static struct rte_mempool *eventdev_func_mempool;
+static int evdev;
+
+struct test {
+	struct rte_mempool *mbuf_pool;
+	int nb_qids;
+};
+
+/* initialization and config */
+static inline int
+init(struct test *t, int nb_queues, int nb_ports)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	memset(t, 0, sizeof(*t));
+
+	t->mbuf_pool = eventdev_func_mempool;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = nb_queues;
+	config.nb_event_ports = nb_ports;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0)
+		printf("%d: Error configuring device\n", __LINE__);
+
+	return ret;
+}
+
+static inline int
+create_ports(int num_ports)
+{
+	int i;
+
+	if (num_ports > MAX_PORTS)
+		return -1;
+
+	for (i = 0; i < num_ports; i++) {
+		struct rte_event_port_conf conf;
+
+		if (rte_event_port_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default port conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		if (rte_event_port_setup(evdev, i, &conf) < 0) {
+			printf("%d: Error setting up port %d\n", i, __LINE__);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static inline int
+create_lb_qids(struct test *t, int num_qids, uint32_t flags)
+{
+	int i;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		struct rte_event_queue_conf conf;
+
+		if (rte_event_queue_default_conf_get(evdev, i, &conf)) {
+			printf("%d: Error querying default queue conf\n",
+			       __LINE__);
+			return -1;
+		}
+
+		conf.schedule_type = flags;
+
+		if (conf.schedule_type == RTE_SCHED_TYPE_PARALLEL)
+			conf.nb_atomic_order_sequences = 0;
+		else
+			conf.nb_atomic_order_sequences = DEFAULT_NUM_SEQ_NUMS;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+static inline int
+create_atomic_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ATOMIC);
+}
+
+static inline int
+create_ordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_ORDERED);
+}
+
+static inline int
+create_unordered_qids(struct test *t, int num_qids)
+{
+	return create_lb_qids(t, num_qids, RTE_SCHED_TYPE_PARALLEL);
+}
+
+static inline int
+create_directed_qids(struct test *t, int num_qids, const uint8_t ports[])
+{
+	static struct rte_event_queue_conf conf;
+	int i;
+
+	conf.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
+	conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = t->nb_qids; i < t->nb_qids + num_qids; i++) {
+		uint8_t queue_id;
+
+		if (rte_event_queue_setup(evdev, i, &conf) < 0) {
+			printf("%d: error creating qid %d\n", __LINE__, i);
+			return -1;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, ports[i - t->nb_qids],
+					&queue_id, NULL, 1) != 1) {
+			printf("%d: error creating link for qid %d\n",
+			       __LINE__, i);
+			return -1;
+		}
+	}
+
+	t->nb_qids += num_qids;
+	if (t->nb_qids > MAX_QIDS)
+		return -1;
+
+	return 0;
+}
+
+/* destruction */
+static inline int
+cleanup(struct test *t __rte_unused)
+{
+	int ret;
+
+	rte_event_dev_stop(evdev);
+	ret = rte_event_dev_close(evdev);
+	if (ret)
+		return -1;
+
+	return 0;
+};
+
+static inline int
+enqueue_timeout(uint8_t port_id, struct rte_event *ev, uint64_t tmo_us)
+{
+	const uint64_t start = rte_get_timer_cycles();
+	const uint64_t ticks = (tmo_us * rte_get_timer_hz()) / 1E6;
+
+	while ((rte_get_timer_cycles() - start) < ticks) {
+		if (rte_event_enqueue_burst(evdev, port_id, ev, 1) == 1)
+			return 0;
+
+		if (rte_errno != -ENOSPC)
+			return -1;
+	}
+
+	return -1;
+}
+
+static void
+flush(uint8_t id __rte_unused, struct rte_event event, void *arg __rte_unused)
+{
+	rte_pktmbuf_free(event.mbuf);
+}
+
+static int
+test_stop_flush(struct test *t) /* test to check we can properly flush events */
+{
+	struct rte_event ev;
+	uint32_t dequeue_depth;
+	unsigned int i, count;
+	uint8_t queue_id;
+
+	ev.op = RTE_EVENT_OP_NEW;
+
+	if (init(t, 2, 1) < 0 ||
+	    create_ports(1) < 0 ||
+	    create_atomic_qids(t, 2) < 0) {
+		printf("%d: Error initializing device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_port_link(evdev, 0, NULL, NULL, 0) != 2) {
+		printf("%d: Error linking queues to the port\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: Error with start call\n", __LINE__);
+		goto err;
+	}
+
+	/* Unlink queue 1 so the PMD's stop callback has to cleanup an unlinked
+	 * queue.
+	 */
+	queue_id = 1;
+
+	if (rte_event_port_unlink(evdev, 0, &queue_id, 1) != 1) {
+		printf("%d: Error unlinking queue 1 from port\n", __LINE__);
+		goto err;
+	}
+
+	count = rte_mempool_avail_count(t->mbuf_pool);
+
+	if (rte_event_port_attr_get(evdev,
+				    0,
+				    RTE_EVENT_PORT_ATTR_DEQ_DEPTH,
+				    &dequeue_depth)) {
+		printf("%d: Error retrieveing dequeue depth\n", __LINE__);
+		goto err;
+	}
+
+	/* Send QEs to queue 0 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 0;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Send QEs to queue 1 */
+	for (i = 0; i < dequeue_depth + 1; i++) {
+		ev.mbuf = rte_pktmbuf_alloc(t->mbuf_pool);
+		ev.queue_id = 1;
+		ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+
+		if (enqueue_timeout(0, &ev, 1000)) {
+			printf("%d: Error enqueuing events\n", __LINE__);
+			goto err;
+		}
+	}
+
+	/* Now the DLB is scheduling events from the port to the IQ, and at
+	 * least one event should be remaining in each queue.
+	 */
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, flush, NULL)) {
+		printf("%d: Error installing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	cleanup(t);
+
+	if (count != rte_mempool_avail_count(t->mbuf_pool)) {
+		printf("%d: Error executing the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dev_stop_flush_callback_register(evdev, NULL, NULL)) {
+		printf("%d: Error uninstalling the flush callback\n", __LINE__);
+		goto err;
+	}
+
+	return 0;
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+test_single_link(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Create a directed port */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) == 0) {
+		printf("%d: port 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Create a load-balanced port */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a directed queue */
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to create another directed queue */
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) == 0) {
+		printf("%d: queue 1 setup expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Create a load-balanced queue */
+	queue_conf.event_queue_cfg = 0;
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	/* Attempt to link directed and load-balanced resources */
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 0 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) == 1) {
+		printf("%d: port 1 link expected to fail\n", __LINE__);
+		goto err;
+	}
+
+	/* Link ports to queues */
+	queue_id = 0;
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	queue_id = 1;
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+#define NUM_LDB_PORTS 64
+#define NUM_LDB_QUEUES 128
+
+static int
+test_info_get(void)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_dev_info info;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	config.nb_event_ports = info.max_event_ports;
+	config.nb_event_queues = NUM_LDB_QUEUES + info.max_event_ports / 2;
+	config.nb_single_link_event_port_queues = info.max_event_ports / 2;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		goto err;
+	}
+
+	/* The DLB PMD only reports load-balanced ports and queues in its
+	 * info_get function. Confirm that these values don't include the
+	 * directed port or queue counts.
+	 */
+
+	if (info.max_event_ports != NUM_LDB_PORTS) {
+		printf("%d: Got %u ports, expected %u\n",
+		       __LINE__, info.max_event_ports, NUM_LDB_PORTS);
+		goto err;
+	}
+
+	if (info.max_event_queues != NUM_LDB_QUEUES) {
+		printf("%d: Got %u queues, expected %u\n",
+		       __LINE__, info.max_event_queues, NUM_LDB_QUEUES);
+		goto err;
+	}
+
+	ret = rte_event_dev_close(evdev);
+	if (ret) {
+		printf("rte_event_dev_close err %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	rte_event_dev_close(evdev);
+	return -1;
+}
+
+static int
+test_reconfiguration_link(struct test *t)
+{
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 2;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 2 LDB queues */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q1, leave P0 and Q0 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 1, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+			printf("%d: P0->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P1->Q0 and Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+			printf("%d: P1->Q%d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Configure device with 2 DIR ports and 2 DIR queues */
+	config.nb_single_link_event_port_queues = 2;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	for (i = 0; i < 2; i++) {
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Link P0->Q0 and P1->Q1 */
+	for (i = 0; i < 2; i++) {
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: port %d link expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	/* Reconfigure device */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure P1 and Q0, leave P0 and Q1 to be configured by the PMD. */
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q1 */
+	queue_id = 1;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: P0->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Link P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: P1->Q%d link expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_queues = 5;
+	config.nb_event_ports = 5;
+	config.nb_single_link_event_port_queues = 1;
+
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	for (i = 0; i < config.nb_event_queues - 1; i++) {
+		port_conf.event_port_cfg = 0;
+		queue_conf.event_queue_cfg = 0;
+
+		if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+			printf("%d: port %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+			printf("%d: queue %d setup expected to succeed\n",
+			       __LINE__, i);
+			goto err;
+		}
+
+		queue_id = i;
+
+		if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+			printf("%d: P%d->Q%d link expected to succeed\n",
+			       __LINE__, i, i);
+			goto err;
+		}
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_SINGLE_LINK;
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, i, &port_conf) < 0) {
+		printf("%d: port %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, i, &queue_conf) < 0) {
+		printf("%d: queue %d setup expected to succeed\n",
+		       __LINE__, i);
+		goto err;
+	}
+
+	queue_id = i;
+
+	if (rte_event_port_link(evdev, i, &queue_id, NULL, 1) != 1) {
+		printf("%d: P%d->Q%d link expected to succeed\n",
+		       __LINE__, i, i);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Stop the device */
+	rte_event_dev_stop(evdev);
+
+	config.nb_event_ports += 1;
+
+	/* Reconfigure device with 1 more load-balanced port */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error re-configuring device\n", __LINE__);
+		return -1;
+	}
+
+	port_conf.event_port_cfg = 0;
+
+	/* Configure the new port */
+	if (rte_event_port_setup(evdev, config.nb_event_ports - 1,
+				 &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	cleanup(t);
+	return 0;
+
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+test_load_balanced_traffic(struct test *t)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup(t);
+	return 0;
+
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+test_directed_traffic(struct test *t)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 1;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 DIR port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.event_port_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 1 NEW event */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 FORWARD event */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (ev.queue_id != 0) {
+		printf("%d: invalid dequeued event queue ID (%d)\n",
+		       __LINE__, ev.queue_id);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_FORWARD;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue and enqueue 1 RELEASE operation */
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: NEW enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	cleanup(t);
+	return 0;
+
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+test_deferred_sched(struct test *t)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	const int num_events = 128;
+	struct rte_event ev;
+	uint8_t queue_id;
+	int ret, i;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 2;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 2 LDB ports and 1 queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 1, DEFERRED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 1;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_setup(evdev, 1, &port_conf) < 0) {
+		printf("%d: port 1 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	queue_conf.schedule_type = RTE_SCHED_TYPE_PARALLEL;
+	queue_conf.nb_atomic_order_sequences = 0;
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 and P1->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_port_link(evdev, 1, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 1 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	/* Enqueue 128 NEW events */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue two events from port 0 (dequeue_depth * 2 due to the
+	 * reserved token scheme)
+	 */
+	timeout = 0xFFFFFFFFF;
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+		printf("%d: event dequeue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Dequeue (and release) all other events from port 1. Deferred
+	 * scheduling ensures no other events are scheduled to port 0 without a
+	 * subsequent rte_event_dequeue_burst() call.
+	 */
+	for (i = 0; i < num_events - 2; i++) {
+		if (rte_event_dequeue_burst(evdev, 1, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		if (rte_event_enqueue_burst(evdev, 1, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup(t);
+	return 0;
+
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+test_delayed_pop(struct test *t)
+{
+	uint64_t timeout;
+	struct rte_event_dev_config config = {0};
+	struct rte_event_queue_conf queue_conf;
+	struct rte_event_port_conf port_conf;
+	struct rte_event_dev_info info;
+	int ret, i, num_events;
+	struct rte_event ev;
+	uint8_t queue_id;
+
+	if (rte_event_dev_info_get(evdev, &info)) {
+		printf("%d: Error querying device info\n", __LINE__);
+		return -1;
+	}
+
+	config.nb_event_queues = 1;
+	config.nb_event_ports = 1;
+	config.nb_single_link_event_port_queues = 0;
+	config.nb_event_queue_flows = info.max_event_queue_flows;
+	config.nb_events_limit = info.max_num_events;
+	config.nb_event_port_dequeue_depth = info.max_event_port_dequeue_depth;
+	config.nb_event_port_enqueue_depth = info.max_event_port_enqueue_depth;
+	config.dequeue_timeout_ns = info.max_dequeue_timeout_ns;
+	config.event_dev_cfg = RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
+
+	/* Configure the device with 1 LDB port and queue */
+	ret = rte_event_dev_configure(evdev, &config);
+	if (ret < 0) {
+		printf("%d: Error configuring device\n", __LINE__);
+		return -1;
+	}
+
+	ret = rte_pmd_dlb_set_token_pop_mode(evdev, 0, DELAYED_POP);
+	if (ret < 0) {
+		printf("%d: Error setting deferred scheduling\n", __LINE__);
+		goto err;
+	}
+
+	/* Configure the ports and queues */
+	if (rte_event_port_default_conf_get(evdev, 0, &port_conf)) {
+		printf("%d: Error querying default port conf\n", __LINE__);
+		goto err;
+	}
+
+	port_conf.dequeue_depth = 16;
+	port_conf.event_port_cfg = RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL;
+
+	if (rte_event_port_setup(evdev, 0, &port_conf) < 0) {
+		printf("%d: port 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_default_conf_get(evdev, 0, &queue_conf)) {
+		printf("%d: Error querying default queue conf\n", __LINE__);
+		goto err;
+	}
+
+	if (rte_event_queue_setup(evdev, 0, &queue_conf) < 0) {
+		printf("%d: queue 0 setup expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Link P0->Q0 */
+	queue_id = 0;
+
+	if (rte_event_port_link(evdev, 0, &queue_id, NULL, 1) != 1) {
+		printf("%d: port 0 link expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	/* Start the device */
+	if (rte_event_dev_start(evdev) < 0) {
+		printf("%d: device start failed\n", __LINE__);
+		goto err;
+	}
+
+	num_events = 2 * port_conf.dequeue_depth;
+
+	/* Enqueue 2 * dequeue_depth NEW events. Due to the PMD's reserved
+	 * token scheme, the port will initially behave as though its
+	 * dequeue_depth is twice the requested size.
+	 */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_PARALLEL;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Flush these events out of the CQ */
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Enqueue 2 * dequeue_depth NEW events again */
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
+	ev.queue_id = 0;
+	ev.priority = 0;
+	ev.u64 = 0;
+
+	for (i = 0; i < num_events; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: NEW enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	/* Dequeue dequeue_depth events but only release dequeue_depth - 2.
+	 * Delayed pop won't perform the pop and no more events will be
+	 * scheduled.
+	 */
+	for (i = 0; i < port_conf.dequeue_depth; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = 0; i < port_conf.dequeue_depth - 2; i++) {
+		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+			printf("%d: RELEASE enqueue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	timeout = 0x10000;
+
+	ret = rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout);
+	if (ret != 0) {
+		printf("%d: event dequeue expected to fail (ret = %d)\n",
+		       __LINE__, ret);
+		goto err;
+	}
+
+	/* Release one more event. This will trigger the token pop, and
+	 * another batch of events will be scheduled to the device.
+	 */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
+		printf("%d: RELEASE enqueue expected to succeed\n",
+		       __LINE__);
+		goto err;
+	}
+
+	timeout = 0xFFFFFFFFF;
+
+	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
+		if (rte_event_dequeue_burst(evdev, 0, &ev, 1, timeout) != 1) {
+			printf("%d: event dequeue expected to succeed\n",
+			       __LINE__);
+			goto err;
+		}
+	}
+
+	cleanup(t);
+	return 0;
+
+err:
+	cleanup(t);
+	return -1;
+}
+
+static int
+do_selftest(void)
+{
+	struct test t;
+	int ret;
+
+	/* Only create mbuf pool once, reuse for each test run */
+	if (!eventdev_func_mempool) {
+		eventdev_func_mempool =
+			rte_pktmbuf_pool_create("EVENTDEV_DLB_SA_MBUF_POOL",
+						(1 << 12), /* 4k buffers */
+						32 /*MBUF_CACHE_SIZE*/,
+						0,
+						512, /* use very small mbufs */
+						rte_socket_id());
+		if (!eventdev_func_mempool) {
+			printf("ERROR creating mempool\n");
+			goto test_fail;
+		}
+	}
+	t.mbuf_pool = eventdev_func_mempool;
+
+	printf("*** Running Stop Flush test...\n");
+	ret = test_stop_flush(&t);
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Single Link test...\n");
+	ret = test_single_link();
+	if (ret != 0) {
+		printf("ERROR - Single Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Info Get test...\n");
+	ret = test_info_get();
+	if (ret != 0) {
+		printf("ERROR - Stop Flush test FAILED.\n");
+		return ret;
+	}
+
+	printf("*** Running Reconfiguration Link test...\n");
+	ret = test_reconfiguration_link(&t);
+	if (ret != 0) {
+		printf("ERROR - Reconfiguration Link test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Load-Balanced Traffic test...\n");
+	ret = test_load_balanced_traffic(&t);
+	if (ret != 0) {
+		printf("ERROR - Load-Balanced Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Directed Traffic test...\n");
+	ret = test_directed_traffic(&t);
+	if (ret != 0) {
+		printf("ERROR - Directed Traffic test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Deferred Scheduling test...\n");
+	ret = test_deferred_sched(&t);
+	if (ret != 0) {
+		printf("ERROR - Deferred Scheduling test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	printf("*** Running Delayed Pop test...\n");
+	ret = test_delayed_pop(&t);
+	if (ret != 0) {
+		printf("ERROR - Delayed Pop test FAILED.\n");
+
+		goto test_fail;
+	}
+
+	return 0;
+
+test_fail:
+	return -1;
+}
+
+int
+test_dlb_eventdev(void)
+{
+	const char *dlb_eventdev_name = "event_dlb";
+	uint8_t num_evdevs = rte_event_dev_count();
+	int i, ret = 0;
+	int found = 0, skipped = 0, passed = 0, failed = 0;
+	struct rte_event_dev_info info;
+
+	for (i = 0; found + skipped < num_evdevs && i < RTE_EVENT_MAX_DEVS;
+	     i++) {
+		ret = rte_event_dev_info_get(i, &info);
+		if (ret < 0)
+			continue;
+
+		/* skip non-dlb event devices */
+		if (strncmp(info.driver_name, dlb_eventdev_name,
+			    sizeof(*info.driver_name)) != 0) {
+			skipped++;
+			continue;
+		}
+
+		evdev = rte_event_dev_get_dev_id(info.driver_name);
+		if (evdev < 0) {
+			printf("Could not get dev_id for eventdev with name %s, i=%d\n",
+			       info.driver_name, i);
+			skipped++;
+			continue;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", info.driver_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       info.driver_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       info.driver_name, ret);
+		}
+	}
+
+	if (found == 0) {
+		printf("No dlb eventdevs found - attempt to create %s\n",
+		       dlb_eventdev_name);
+		if (rte_vdev_init(dlb_eventdev_name, NULL) < 0) {
+			printf("Error creating eventdev %s\n",
+			       dlb_eventdev_name);
+			goto test_fail;
+		}
+		evdev = rte_event_dev_get_dev_id(dlb_eventdev_name);
+		if (evdev < 0) {
+			printf("Error finding newly created eventdev %s\n",
+			       dlb_eventdev_name);
+			goto test_fail;
+		}
+		found++;
+		printf("Running selftest on eventdev %s\n", dlb_eventdev_name);
+		ret = do_selftest();
+		if (ret == 0) {
+			passed++;
+			printf("Selftest passed for eventdev %s\n",
+			       dlb_eventdev_name);
+		} else {
+			failed++;
+			printf("Selftest failed for eventdev %s, err=%d\n",
+			       dlb_eventdev_name, ret);
+		}
+	}
+
+test_fail:
+	printf("Ran selftest on %d eventdevs, %d skipped, %d passed, %d failed\n",
+	       found, skipped, passed, failed);
+	return ret;
+}
-- 
2.13.6

[dpdk-dev] [PATCH 15/27] event/dlb: add probe

[McDaniel, Timothy]

Change-Id: I4b8d0ca2bd5eb01d4ea8c4c6c9487b4f7116d68e
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 519 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 519 insertions(+)
 create mode 100644 drivers/event/dlb/dlb.c

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 000000000..124b86a1d
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,519 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <nmmintrin.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_string_fns.h>
+#include <rte_prefetch.h>
+
+#include "dlb_priv.h"
+#include "dlb_iface.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev. Some values overridden at runtime using
+ * values returned by the DLB kernel driver.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
+
+/* These functions will vary based on processor capabilities */
+static struct dlb_port_low_level_io_functions qm_mmio_fns;
+
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	/* Query driver resources provisioned for this VF */
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("ioctl get dlb num resources, err=%d\n",
+			    ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define RTE_BASE_10 10
+int dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, NULL, RTE_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+static void
+dlb_qm_mmio_fn_init(void)
+{
+	/* Process-local function pointers for performing low level port i/o */
+
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_MOVDIR64B))
+		qm_mmio_fns.pp_enqueue_four = dlb_movdir64b;
+	else
+		qm_mmio_fns.pp_enqueue_four = dlb_movntdq;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args,
+			   bool is_vdev)
+{
+	struct dlb_eventdev *dlb;
+	int err, i;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+	dlb->is_vdev = is_vdev; /* vdev or pf */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->qm_instance.device_path_id = dlb_args->dev_id;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n",
+			    err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n",
+			    err);
+		return err;
+	}
+
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
+	/* Initialize each port's token pop mode */
+	for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+		dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_qm_mmio_fn_init();
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name,
+			     bool is_vdev)
+{
+	struct dlb_eventdev *dlb;
+	int err;
+
+RTE_SET_USED(is_vdev);
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n",
+			    err, name);
+		return err;
+	}
+
+	dlb_qm_mmio_fn_init();
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params != NULL && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (!kvlist) {
+			RTE_LOG(INFO, PMD,
+				"Ignoring unsupported parameters when creating device '%s'\n",
+				name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
+
+/* declared extern in header, for access from other .c files */
+int eventdev_dlb_log_level;
+
+RTE_INIT(evdev_dlb_init_log)
+{
+	eventdev_dlb_log_level = rte_log_register("pmd.event.dlb");
+	if (eventdev_dlb_log_level >= 0)
+		rte_log_set_level(eventdev_dlb_log_level, RTE_LOG_NOTICE);
+}
-- 
2.13.6

[dpdk-dev] [PATCH 16/27] event/dlb: add infos_get and configure

[McDaniel, Timothy]

Change-Id: I749ae914852bc3516601301c2b1fb338ba7508f7
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 401 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 401 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 124b86a1d..c21ebe7e2 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -220,6 +220,394 @@ set_num_dir_credits(const char *key __rte_unused,
 			    DLB_MAX_NUM_DIR_CREDITS);
 		return -EINVAL;
 	}
+	return 0;
+}
+
+/* VDEV-only notes:
+ * This function first unmaps all memory mappings and closes the
+ * domain's file descriptor, which causes the driver to reset the
+ * scheduling domain. Once that completes (when close() returns), we
+ * can safely free the dynamically allocated memory used by the
+ * scheduling domain.
+ *
+ * PF-only notes:
+ * We will maintain a use count and use that to determine when
+ * a reset is required.  In PF mode, we never mmap, or munmap
+ * device memory,  and we own the entire physical PCI device.
+ */
+
+static void
+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	enum dlb_configuration_state config_state;
+	int i, j;
+
+	/* Close and reset the domain */
+	dlb_iface_domain_close(dlb);
+
+	/* Free all dynamically allocated port memory */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_free_qe_mem(&dlb->ev_ports[i].qm_port);
+
+	/* If reconfiguring, mark the device's queues and ports as "previously
+	 * configured." If the user doesn't reconfigure them, the PMD will
+	 * reapply their previous configuration when the device is started.
+	 */
+	config_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		dlb->ev_ports[i].qm_port.config_state = config_state;
+		/* Reset setup_done so ports can be reconfigured */
+		dlb->ev_ports[i].setup_done = false;
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			dlb->ev_ports[i].link[j].mapped = false;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++)
+		dlb->ev_queues[i].qm_queue.config_state = config_state;
+
+	for (i = 0; i < DLB_MAX_NUM_QUEUES; i++)
+		dlb->ev_queues[i].setup_done = false;
+
+	dlb->num_ports = 0;
+	dlb->num_ldb_ports = 0;
+	dlb->num_dir_ports = 0;
+	dlb->num_queues = 0;
+	dlb->num_ldb_queues = 0;
+	dlb->num_dir_queues = 0;
+	dlb->configured = false;
+}
+
+static int
+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_ldb_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_ldb_credits) {
+		handle->cfg.ldb_credit_pool_id = 0;
+		handle->cfg.num_ldb_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;
+
+	ret = dlb_iface_ldb_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: ldb_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+	}
+
+	handle->cfg.ldb_credit_pool_id = response.id;
+	handle->cfg.num_ldb_credits = cfg.num_ldb_credits;
+
+	return ret;
+}
+
+static int
+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)
+{
+	struct dlb_create_dir_pool_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (!handle->cfg.resources.num_dir_credits) {
+		handle->cfg.dir_credit_pool_id = 0;
+		handle->cfg.num_dir_credits = 0;
+		return 0;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_dir_credits = handle->cfg.resources.num_dir_credits;
+
+	ret = dlb_iface_dir_credit_pool_create(handle,
+					       &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: dir_credit_pool_create ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	handle->cfg.dir_credit_pool_id = response.id;
+	handle->cfg.num_dir_credits = cfg.num_dir_credits;
+
+	return ret;
+}
+
+static int
+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
+			   struct dlb_eventdev *dlb,
+			   const struct dlb_hw_rsrcs *resources_asked)
+{
+	int ret = 0;
+	struct dlb_create_sched_domain_args *config_params;
+	struct dlb_cmd_response response;
+
+	if (resources_asked == NULL) {
+		DLB_LOG_ERR("dlb: dlb_create NULL parameter\n");
+		ret = EINVAL;
+		goto error_exit;
+	}
+
+	/* Map generic qm resources to dlb resources */
+	config_params = &handle->cfg.resources;
+
+	config_params->response = (uintptr_t)&response;
+
+	/* DIR ports and queues */
+
+	config_params->num_dir_ports =
+		resources_asked->num_dir_ports;
+
+	config_params->num_dir_credits =
+		resources_asked->num_dir_credits;
+
+	/* LDB ports and queues */
+
+	config_params->num_ldb_queues =
+		resources_asked->num_ldb_queues;
+
+	config_params->num_ldb_ports =
+		resources_asked->num_ldb_ports;
+
+	config_params->num_ldb_credits =
+		resources_asked->num_ldb_credits;
+
+	config_params->num_atomic_inflights =
+		dlb->num_atm_inflights_per_queue *
+		config_params->num_ldb_queues;
+
+	config_params->num_hist_list_entries = config_params->num_ldb_ports *
+		DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* dlb limited to 1 credit pool per queue type */
+	config_params->num_ldb_credit_pools = 1;
+	config_params->num_dir_credit_pools = 1;
+
+	DLB_LOG_DBG("sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\n",
+		    config_params->num_ldb_queues,
+		    config_params->num_ldb_ports,
+		    config_params->num_dir_ports,
+		    config_params->num_atomic_inflights,
+		    config_params->num_hist_list_entries,
+		    config_params->num_ldb_credits,
+		    config_params->num_dir_credits,
+		    config_params->num_ldb_credit_pools,
+		    config_params->num_dir_credit_pools);
+
+	/* Configure the QM */
+
+	ret = dlb_iface_sched_domain_create(handle, config_params);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\n",
+			    handle->device_id,
+			    ret,
+			    dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	handle->domain_id = response.id;
+	handle->domain_id_valid = 1;
+
+	config_params->response = 0;
+
+	ret = dlb_ldb_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create ldb credit pool failed\n");
+		goto error_exit2;
+	}
+
+	ret = dlb_dir_credit_pool_create(handle);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create dir credit pool failed\n");
+		goto error_exit2;
+	}
+
+	handle->cfg.configured = true;
+
+	return 0;
+
+error_exit2:
+	dlb_iface_domain_close(dlb);
+
+error_exit:
+
+	return ret;
+}
+
+/* End HW specific */
+static void
+dlb_eventdev_info_get(struct rte_eventdev *dev,
+		      struct rte_event_dev_info *dev_info)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret;
+
+	ret = dlb_hw_query_resources(dlb);
+	if (ret) {
+		const struct rte_eventdev_data *data = dev->data;
+
+		DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+			    ret, data->dev_id);
+		/* fn is void, so fall through and return values set up in
+		 * probe
+		 */
+	}
+
+	/* Add num resources currently owned by this domain.
+	 * These would become available if the scheduling domain were reset due
+	 * to the application recalling eventdev_configure to *reconfigure* the
+	 * domain.
+	 */
+	evdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;
+	evdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;
+	evdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;
+
+	/* In DLB A-stepping hardware, applications are limited to 128
+	 * configured ports (load-balanced or directed). The reported number of
+	 * available ports must reflect this.
+	 */
+	if (dlb->revision < DLB_REV_B0) {
+		int used_ports;
+
+		used_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -
+			dlb->hw_rsrc_query_results.num_ldb_ports -
+			dlb->hw_rsrc_query_results.num_dir_ports;
+
+		evdev_dlb_default_info.max_event_ports =
+			RTE_MIN(evdev_dlb_default_info.max_event_ports,
+				128 - used_ports);
+	}
+
+	evdev_dlb_default_info.max_event_queues =
+		RTE_MIN(evdev_dlb_default_info.max_event_queues,
+			RTE_EVENT_MAX_QUEUES_PER_DEV);
+
+	evdev_dlb_default_info.max_num_events =
+		RTE_MIN(evdev_dlb_default_info.max_num_events,
+			dlb->max_num_events_override);
+
+	*dev_info = evdev_dlb_default_info;
+}
+
+/* Note: 1 QM instance per QM device, QM instance/device == event device */
+static int
+dlb_eventdev_configure(const struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;
+	const struct rte_eventdev_data *data = dev->data;
+	const struct rte_event_dev_config *config = &data->dev_conf;
+	int ret;
+
+	/* If this eventdev is already configured, we must release the current
+	 * scheduling domain before attempting to configure a new one.
+	 */
+	if (dlb->configured) {
+		dlb_hw_reset_sched_domain(dev, true);
+
+		ret = dlb_hw_query_resources(dlb);
+		if (ret) {
+			DLB_LOG_ERR("get resources err=%d, devid=%d\n",
+				    ret, data->dev_id);
+			return ret;
+		}
+	}
+
+	if (config->nb_event_queues > rsrcs->num_queues) {
+		DLB_LOG_ERR("nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_queues,
+			    rsrcs->num_queues);
+		return -EINVAL;
+	}
+	if (config->nb_event_ports > (rsrcs->num_ldb_ports
+			+ rsrcs->num_dir_ports)) {
+		DLB_LOG_ERR("nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_event_ports,
+			    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));
+		return -EINVAL;
+	}
+	if (config->nb_events_limit > rsrcs->nb_events_limit) {
+		DLB_LOG_ERR("nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\n",
+			    config->nb_events_limit,
+			    rsrcs->nb_events_limit);
+		return -EINVAL;
+	}
+
+	if (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
+		dlb->global_dequeue_wait = false;
+	} else {
+		uint32_t timeout32;
+
+		dlb->global_dequeue_wait = true;
+
+		timeout32 = config->dequeue_timeout_ns;
+
+		/* PF PMD does not support interrupts, and
+		 * UMONITOR wait is temporarily disabled.
+		 */
+
+		dlb->global_dequeue_wait_ticks =
+			timeout32 * (rte_get_timer_hz() / 1E9);
+	}
+
+	/* Does this platform support umonitor/umwait? */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_UMWAIT)) {
+		if (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&
+		    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {
+			DLB_LOG_ERR("invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\n",
+				    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);
+			return -EINVAL;
+		}
+		dlb->umwait_allowed = true;
+	}
+
+	rsrcs->num_dir_ports = config->nb_single_link_event_port_queues;
+	rsrcs->num_ldb_ports  = config->nb_event_ports - rsrcs->num_dir_ports;
+	/* 1 dir queue per dir port */
+	rsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;
+
+	/* Scale down nb_events_limit by 4 for directed credits, since there
+	 * are 4x as many load-balanced credits.
+	 */
+	rsrcs->num_ldb_credits = 0;
+	rsrcs->num_dir_credits = 0;
+
+	if (rsrcs->num_ldb_queues)
+		rsrcs->num_ldb_credits = config->nb_events_limit;
+	if (rsrcs->num_dir_ports)
+		rsrcs->num_dir_credits = config->nb_events_limit / 4;
+	if (dlb->num_dir_credits_override != -1)
+		rsrcs->num_dir_credits = dlb->num_dir_credits_override;
+
+	if (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {
+		DLB_LOG_ERR("dlb_hw_create_sched_domain failed\n");
+		return -ENODEV;
+	}
+
+	dlb->new_event_limit = config->nb_events_limit;
+	rte_atomic32_set(&dlb->inflights, 0);
+
+	/* Save number of ports/queues for this event dev */
+	dlb->num_ports = config->nb_event_ports;
+	dlb->num_queues = config->nb_event_queues;
+	dlb->num_dir_ports = rsrcs->num_dir_ports;
+	dlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;
+	dlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;
+	dlb->num_dir_queues = dlb->num_dir_ports;
+	dlb->num_ldb_credits = rsrcs->num_ldb_credits;
+	dlb->num_dir_credits = rsrcs->num_dir_credits;
+
+	dlb->configured = true;
 
 	return 0;
 }
@@ -294,6 +682,19 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dev_infos_get    = dlb_eventdev_info_get,
+		.dev_configure    = dlb_eventdev_configure,
+	};
+
+	/* Expose PMD's eventdev interface */
+
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+
+}
 
 static void
 dlb_qm_mmio_fn_init(void)
-- 
2.13.6

[dpdk-dev] [PATCH 17/27] event/dlb: add queue_def_conf and port_def_conf

[McDaniel, Timothy]

Change-Id: Ifa154a041fa117776612efc027b128c1603b6396
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index c21ebe7e2..adaeafedc 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -612,6 +612,35 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb;
+
+	dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -688,6 +717,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 	};
 
 	/* Expose PMD's eventdev interface */
-- 
2.13.6

[dpdk-dev] [PATCH 18/27] event/dlb: add queue setup

[McDaniel, Timothy]

Change-Id: I88389542d5dbb64b255ce06605ace9d3315f7b22
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 295 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 295 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index adaeafedc..2e8000047 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -223,6 +223,65 @@ set_num_dir_credits(const char *key __rte_unused,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
+			struct dlb_queue *queue,
+			const struct rte_event_queue_conf *evq_conf)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+	uint32_t qm_qid;
+	int sched_type = -1;
+
+	if (evq_conf == NULL)
+		return -EINVAL;
+
+	if (evq_conf->event_queue_cfg & RTE_EVENT_QUEUE_CFG_ALL_TYPES) {
+		if (evq_conf->nb_atomic_order_sequences != 0)
+			sched_type = RTE_SCHED_TYPE_ORDERED;
+		else
+			sched_type = RTE_SCHED_TYPE_PARALLEL;
+	} else {
+		sched_type = evq_conf->schedule_type;
+	}
+
+	cfg.response = (uintptr_t)&response;
+	cfg.num_atomic_inflights = dlb->num_atm_inflights_per_queue;
+	cfg.num_sequence_numbers = evq_conf->nb_atomic_order_sequences;
+	cfg.num_qid_inflights = evq_conf->nb_atomic_order_sequences;
+
+	if (sched_type != RTE_SCHED_TYPE_ORDERED) {
+		cfg.num_sequence_numbers = 0;
+		cfg.num_qid_inflights = 2048;
+	}
+
+	ret = dlb_iface_ldb_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create LB event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	qm_qid = response.id;
+
+	/* Save off queue config for debug, resource lookups, and reconfig */
+	queue->num_qid_inflights = cfg.num_qid_inflights;
+	queue->num_atm_inflights = cfg.num_atomic_inflights;
+
+	queue->sched_type = sched_type;
+	queue->config_state = DLB_CONFIGURED;
+
+	DLB_LOG_DBG("Created LB event queue %d, nb_inflights=%d, nb_seq=%d, qid inflights=%d\n",
+		    qm_qid,
+		    cfg.num_atomic_inflights,
+		    cfg.num_sequence_numbers,
+		    cfg.num_qid_inflights);
+
+	return qm_qid;
+}
+
 /* VDEV-only notes:
  * This function first unmaps all memory mappings and closes the
  * domain's file descriptor, which causes the driver to reset the
@@ -444,6 +503,7 @@ dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,
 }
 
 /* End HW specific */
+
 static void
 dlb_eventdev_info_get(struct rte_eventdev *dev,
 		      struct rte_event_dev_info *dev_info)
@@ -641,6 +701,240 @@ dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
 	queue_conf->priority = 0;
 }
 
+static int32_t
+dlb_get_sn_allocation(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_set_sn_allocation(struct dlb_eventdev *dlb, int group, int num)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_set_sn_allocation_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.num = num;
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_set_sn_allocation(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: set_sn_allocation ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int32_t
+dlb_get_sn_occupancy(struct dlb_eventdev *dlb, int group)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_sn_occupancy_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.group = group;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_sn_occupancy(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_sn_occupancy ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+/* Query the current sequence number allocations and, if they conflict with the
+ * requested LDB queue configuration, attempt to re-allocate sequence numbers.
+ * This is best-effort; if it fails, the PMD will attempt to configure the
+ * load-balanced queue and return an error.
+ */
+static void
+dlb_program_sn_allocation(struct dlb_eventdev *dlb,
+			  const struct rte_event_queue_conf *queue_conf)
+{
+	int grp_occupancy[DLB_NUM_SN_GROUPS];
+	int grp_alloc[DLB_NUM_SN_GROUPS];
+	int i, sequence_numbers;
+
+	sequence_numbers = (int)queue_conf->nb_atomic_order_sequences;
+
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		int total_slots;
+
+		grp_alloc[i] = dlb_get_sn_allocation(dlb, i);
+		if (grp_alloc[i] < 0)
+			return;
+
+		total_slots = DLB_MAX_LDB_SN_ALLOC / grp_alloc[i];
+
+		grp_occupancy[i] = dlb_get_sn_occupancy(dlb, i);
+		if (grp_occupancy[i] < 0)
+			return;
+
+		/* DLB has at least one available slot for the requested
+		 * sequence numbers, so no further configuration required.
+		 */
+		if (grp_alloc[i] == sequence_numbers &&
+		    grp_occupancy[i] < total_slots)
+			return;
+	}
+
+	/* None of the sequence number groups are configured for the requested
+	 * sequence numbers, so we have to reconfigure one of them. This is
+	 * only possible if a group is not in use.
+	 */
+	for (i = 0; i < DLB_NUM_SN_GROUPS; i++) {
+		if (grp_occupancy[i] == 0)
+			break;
+	}
+
+	if (i == DLB_NUM_SN_GROUPS) {
+		printf("[%s()] No groups with %d sequence_numbers are available or have free slots\n",
+		       __func__, sequence_numbers);
+		return;
+	}
+
+	/* Attempt to configure slot i with the requested number of sequence
+	 * numbers. Ignore the return value -- if this fails, the error will be
+	 * caught during subsequent queue configuration.
+	 */
+	dlb_set_sn_allocation(dlb, i, sequence_numbers);
+}
+
+static int
+dlb_eventdev_ldb_queue_setup(struct rte_eventdev *dev,
+			     struct dlb_eventdev_queue *ev_queue,
+			     const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int32_t qm_qid;
+
+	if (queue_conf->nb_atomic_order_sequences)
+		dlb_program_sn_allocation(dlb, queue_conf);
+
+	qm_qid = dlb_hw_create_ldb_queue(dlb,
+					 &ev_queue->qm_queue,
+					 queue_conf);
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the load-balanced queue\n");
+
+		return qm_qid;
+	}
+
+	dlb->qm_ldb_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)
+{
+	int i, num = 0;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].setup_done &&
+		    dlb->ev_queues[i].qm_queue.is_directed)
+			num++;
+	}
+
+	return num;
+}
+
+static void
+dlb_queue_link_teardown(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *ev_queue)
+{
+	struct dlb_eventdev_port *ev_port;
+	int i, j;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			if (!ev_port->link[j].valid ||
+			    ev_port->link[j].queue_id != ev_queue->id)
+				continue;
+
+			ev_port->link[j].valid = false;
+			ev_port->num_links--;
+		}
+	}
+
+	ev_queue->num_links = 0;
+}
+
+static int
+dlb_eventdev_queue_setup(struct rte_eventdev *dev,
+			 uint8_t ev_qid,
+			 const struct rte_event_queue_conf *queue_conf)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_queue *ev_queue;
+	int ret;
+
+	if (!queue_conf)
+		return -EINVAL;
+
+	if (ev_qid >= dlb->num_queues)
+		return -EINVAL;
+
+	ev_queue = &dlb->ev_queues[ev_qid];
+
+	ev_queue->qm_queue.is_directed = queue_conf->event_queue_cfg &
+		RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
+	ev_queue->id = ev_qid;
+	ev_queue->conf = *queue_conf;
+
+	if (!ev_queue->qm_queue.is_directed) {
+		ret = dlb_eventdev_ldb_queue_setup(dev, ev_queue, queue_conf);
+	} else {
+		/* The directed queue isn't setup until link time, at which
+		 * point we know its directed port ID. Directed queue setup
+		 * will only fail if this queue is already setup or there are
+		 * no directed queues left to configure.
+		 */
+		ret = 0;
+
+		ev_queue->qm_queue.config_state = DLB_NOT_CONFIGURED;
+
+		if (ev_queue->setup_done ||
+		    dlb_num_dir_queues_setup(dlb) == dlb->num_dir_queues)
+			ret = -EINVAL;
+	}
+
+	/* Tear down pre-existing port->queue links */
+	if (!ret && dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_queue_link_teardown(dlb, ev_queue);
+
+	if (!ret)
+		ev_queue->setup_done = true;
+
+	return ret;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -718,6 +1012,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 	};
 
-- 
2.13.6

[dpdk-dev] [PATCH 19/27] event/dlb: add port_setup

[McDaniel, Timothy]

Change-Id: Ibfa0ca4c9e19c0443e199e34eb989a5e959c2a0b
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 1241 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1241 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2e8000047..deea474bf 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -71,6 +71,29 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 /* These functions will vary based on processor capabilities */
 static struct dlb_port_low_level_io_functions qm_mmio_fns;
 
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+static __rte_always_inline uint16_t
+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)
+{
+	volatile uint16_t *popcount;
+
+	if (ldb)
+		popcount = port_data->ldb_popcount;
+	else
+		popcount = port_data->dir_popcount;
+
+	return *popcount;
+}
+
+static __rte_always_inline void
+dlb_pp_write(struct dlb_enqueue_qe *qe4,
+	     struct process_local_port_data *port_data)
+{
+	qm_mmio_fns.pp_enqueue_four(qe4, port_data->pp_addr);
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -171,6 +194,46 @@ set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+static inline int
+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_cq_pop_qe *qe;
+
+	RTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return 0;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe = qm_port->consume_qe;
+
+	qe->tokens = num - 1;
+	qe->int_arm = 0;
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_movntdq_single(qe, port_data->pp_addr);
+
+	DLB_LOG_DBG("dlb: consume immediate - %d QEs\n", num);
+
+	qm_port->owed_tokens = 0;
+
+	return 0;
+}
+
 int
 set_max_num_events(const char *key __rte_unused,
 		   const char *value,
@@ -223,6 +286,455 @@ set_num_dir_credits(const char *key __rte_unused,
 	return 0;
 }
 
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			   const struct rte_event events[]);
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				 const struct rte_event events[],
+				 uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				     const struct rte_event events[],
+				     uint16_t num);
+
+static inline uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					 const struct rte_event events[],
+					 uint16_t num);
+
+int
+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
+{
+	int ret, sz;
+
+	sz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);
+
+	qm_port->qe4 = rte_malloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+	if (qm_port->qe4 == NULL) {
+		DLB_LOG_ERR("dlb: no qe4 memory\n");
+		ret = -ENOMEM;
+		goto error_exit;
+	}
+
+	memset(qm_port->qe4, 0, sz);
+
+	ret = dlb_init_consume_qe(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_init_consume_qe ret=%d\n",
+			    ret);
+		goto error_exit;
+	}
+
+	return 0;
+
+error_exit:
+
+	dlb_free_qe_mem(qm_port);
+
+	return ret;
+}
+
+int
+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)
+{
+	struct dlb_cq_pop_qe *qe;
+
+	qe = rte_malloc(mz_name,
+			DLB_NUM_QES_PER_CACHE_LINE *
+				sizeof(struct dlb_cq_pop_qe),
+			RTE_CACHE_LINE_SIZE);
+
+	if (qe == NULL)	{
+		DLB_LOG_ERR("dlb: no memory for consume_qe\n");
+		return -ENOMEM;
+	}
+
+	qm_port->consume_qe = qe;
+
+	memset(qe, 0, DLB_NUM_QES_PER_CACHE_LINE *
+	       sizeof(struct dlb_cq_pop_qe));
+
+	qe->qe_valid = 0;
+	qe->qe_frag = 0;
+	qe->qe_comp = 0;
+	qe->cq_token = 1;
+	/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+	 * and so on.
+	 */
+	qe->tokens = 0;	/* set at run time */
+	qe->meas_lat = 0;
+	qe->no_dec = 0;
+	/* Completion IDs are disabled */
+	qe->cmp_id = 0;
+
+	return 0;
+}
+
+static int
+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_ldb_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_LDB_CQ_DEPTH ||
+	    cq_depth > DLB_MAX_INPUT_QUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			DLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	if (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid enqueue_depth, must be at least %d\n",
+			    DLB_MIN_ENQUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	cfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* If there are no directed ports, the kernel driver will ignore this
+	 * port's directed credit settings. Don't use enqueue_depth if it would
+	 * require more directed credits than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	/* DEBUG
+	 * DLB_LOG_ERR("create ldb port - grp=%d, devId=%d\n",
+	 * handle->cfg.domain_id, handle->device_id);
+	 */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm LB port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb; /* back ptr */
+	/*
+	 * Allocate and init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "%s_ldb_port%d",
+		 handle->device_name,
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	/* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+	 * the effective depth is smaller.
+	 */
+	qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];
+
+	/* When using the reserved token scheme, token_pop_thresh is
+	 * initially 2 * dequeue_depth. Once the tokens are reserved,
+	 * the enqueue code re-assigns it to dequeue_depth.
+	 */
+	qm_port->dequeue_depth = dequeue_depth;
+	qm_port->token_pop_thresh = cq_depth;
+
+	/* When the deferred scheduling vdev arg is selected, use deferred pop
+	 * for all single-entry CQs.
+	 */
+	if (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {
+		if (dlb->defer_sched)
+			qm_port->token_pop_mode = DEFERRED_POP;
+	}
+
+	/* The default enqueue functions do not include delayed-pop support for
+	 * performance reasons.
+	 */
+	if (qm_port->token_pop_mode == DELAYED_POP) {
+		dlb->event_dev->enqueue = dlb_event_enqueue_delayed;
+		dlb->event_dev->enqueue_burst =
+			dlb_event_enqueue_burst_delayed;
+		dlb->event_dev->enqueue_new_burst =
+			dlb_event_enqueue_new_burst_delayed;
+		dlb->event_dev->enqueue_forward_burst =
+			dlb_event_enqueue_forward_burst_delayed;
+	}
+
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* Save config message too. */
+	rte_memcpy(&qm_port->cfg.ldb, &cfg, sizeof(cfg));
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    qm_port->ldb_credits,
+		    qm_port->dir_credits);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+
+	if (qm_port) {
+		dlb_free_qe_mem(qm_port);
+		qm_port->pp_mmio_base = 0;
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create ldb port failed!\n");
+
+	return ret;
+}
+
+static int
+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port,
+		       uint32_t dequeue_depth,
+		       uint32_t cq_depth,
+		       uint32_t enqueue_depth,
+		       uint16_t rsvd_tokens,
+		       bool use_rsvd_token_scheme)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_port_args cfg = {0};
+	struct dlb_cmd_response response = {0};
+	int ret;
+	struct dlb_port *qm_port = NULL;
+	char mz_name[RTE_MEMZONE_NAMESIZE];
+	uint32_t qm_port_id;
+
+	if (dlb == NULL || handle == NULL)
+		return -EINVAL;
+
+	if (cq_depth < DLB_MIN_DIR_CQ_DEPTH ||
+	    cq_depth > DLB_MAX_INPUT_QUEUE_DEPTH) {
+		DLB_LOG_ERR("dlb: invalid cq_depth, must be %d-%d\n",
+			    DLB_MIN_DIR_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&handle->resource_lock);
+
+	/* Directed queues are configured at link time. */
+	cfg.queue_id = -1;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* We round up to the next power of 2 if necessary */
+	cfg.cq_depth = rte_align32pow2(cq_depth);
+	cfg.cq_depth_threshold = rsvd_tokens;
+
+	/* User controls the LDB high watermark via enqueue depth. The DIR high
+	 * watermark is equal, unless the directed credit pool is too small.
+	 */
+	cfg.ldb_credit_high_watermark = enqueue_depth;
+
+	/* Don't use enqueue_depth if it would require more directed credits
+	 * than are available.
+	 */
+	cfg.dir_credit_high_watermark =
+		RTE_MIN(enqueue_depth,
+			handle->cfg.num_dir_credits / dlb->num_ports);
+
+	cfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;
+	cfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);
+
+	cfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;
+	cfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);
+
+	/* Per QM values */
+
+	cfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;
+	cfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;
+
+	ret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		goto error_exit;
+	}
+
+	qm_port_id = response.id;
+
+	DLB_LOG_DBG("dlb: ev_port %d uses qm DIR port %d <<<<<\n",
+		    ev_port->id, qm_port_id);
+
+	qm_port = &ev_port->qm_port;
+	qm_port->ev_port = ev_port; /* back ptr */
+	qm_port->dlb = dlb;  /* back ptr */
+
+	/*
+	 * Init local qe struct(s).
+	 * Note: MOVDIR64 requires the enqueue QE to be aligned
+	 */
+
+	snprintf(mz_name, sizeof(mz_name), "%s_dir_port%d",
+		 handle->device_name,
+		 ev_port->id);
+
+	ret = dlb_init_qe_mem(qm_port, mz_name);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: init_qe_mem failed, ret=%d\n", ret);
+		goto error_exit;
+	}
+
+	qm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;
+	qm_port->id = qm_port_id;
+
+	/* The credit window is one high water mark of QEs */
+	qm_port->ldb_pushcount_at_credit_expiry = 0;
+	qm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;
+	/* The credit window is one high water mark of QEs */
+	qm_port->dir_pushcount_at_credit_expiry = 0;
+	qm_port->cached_dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->cq_depth = cfg.cq_depth;
+	qm_port->cq_idx = 0;
+	qm_port->cq_idx_unmasked = 0;
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)
+		qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+	else
+		qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+	qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+	/* starting value of gen bit - it toggles at wrap time */
+	qm_port->gen_bit = 1;
+
+	qm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;
+	qm_port->cq_rsvd_token_deficit = rsvd_tokens;
+	qm_port->int_armed = false;
+
+	/* Save off for later use in info and lookup APIs. */
+	qm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];
+
+	qm_port->dequeue_depth = dequeue_depth;
+
+	/* Directed ports are auto-pop, by default. */
+	qm_port->token_pop_mode = AUTO_POP;
+	qm_port->owed_tokens = 0;
+	qm_port->issued_releases = 0;
+
+	/* Save config message too. */
+	rte_memcpy(&qm_port->cfg.dir, &cfg, sizeof(cfg));
+
+	/* update state */
+	qm_port->state = PORT_STARTED; /* enabled at create time */
+	qm_port->config_state = DLB_CONFIGURED;
+
+	qm_port->dir_credits = cfg.dir_credit_high_watermark;
+	qm_port->ldb_credits = cfg.ldb_credit_high_watermark;
+
+	DLB_LOG_DBG("dlb: created dir port %d, depth = %d cr=%d,%d\n",
+		    qm_port_id,
+		    cq_depth,
+		    cfg.dir_credit_high_watermark,
+		    cfg.ldb_credit_high_watermark);
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	return 0;
+
+error_exit:
+
+	if (qm_port) {
+		qm_port->pp_mmio_base = 0;
+		dlb_free_qe_mem(qm_port);
+	}
+
+	rte_spinlock_unlock(&handle->resource_lock);
+
+	DLB_LOG_ERR("dlb: create dir port failed!\n");
+
+	return ret;
+}
+
 static int32_t
 dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 			struct dlb_queue *queue,
@@ -282,6 +794,15 @@ dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 	return qm_qid;
 }
 
+static inline void
+dlb_hw_do_enqueue(struct dlb_port *qm_port,
+		  struct process_local_port_data *port_data)
+{
+	DLB_LOG_DBG("dlb: Flushing QE(s) to DLB\n");
+
+	dlb_pp_write(qm_port->qe4, port_data);
+}
+
 /* VDEV-only notes:
  * This function first unmaps all memory mappings and closes the
  * domain's file descriptor, which causes the driver to reset the
@@ -558,6 +1079,598 @@ dlb_eventdev_info_get(struct rte_eventdev *dev,
 	*dev_info = evdev_dlb_default_info;
 }
 
+static inline int
+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_port *ev_port)
+{
+	uint32_t sw_inflights = rte_atomic32_read(&dlb->inflights);
+	const int num = 1;
+
+	if (unlikely(ev_port->inflight_max < sw_inflights)) {
+		DLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);
+		rte_errno = -ENOSPC;
+		return 1;
+	}
+
+	if (ev_port->inflight_credits < num) {
+		/* check if event enqueue brings ev_port over max threshold */
+		uint32_t credit_update_quanta = ev_port->credit_update_quanta;
+
+		if (sw_inflights + credit_update_quanta >
+				dlb->new_event_limit) {
+			DLB_INC_STAT(
+				ev_port->stats.traffic.tx_nospc_new_event_limit,
+				1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+
+		rte_atomic32_add(&dlb->inflights, credit_update_quanta);
+		ev_port->inflight_credits += (credit_update_quanta);
+
+		if (ev_port->inflight_credits < num) {
+			DLB_INC_STAT(
+			    ev_port->stats.traffic.tx_nospc_inflight_credits,
+			    1);
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static inline void
+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port)
+{
+	uint16_t quanta = ev_port->credit_update_quanta;
+
+	if (ev_port->inflight_credits >= quanta * 2) {
+		rte_atomic32_sub(&dlb->inflights, quanta);
+		ev_port->inflight_credits -= quanta;
+	}
+}
+
+static inline int
+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_ldb_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, true);
+
+		qm_port->cached_ldb_credits = pc -
+			qm_port->ldb_pushcount_at_credit_expiry;
+		if (unlikely(qm_port->cached_ldb_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,
+			1);
+
+			DLB_LOG_DBG("ldb credits exhausted\n");
+			return 1;
+		}
+		qm_port->ldb_pushcount_at_credit_expiry +=
+			qm_port->cached_ldb_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,
+				 struct process_local_port_data *port_data)
+{
+	if (unlikely(qm_port->cached_dir_credits == 0)) {
+		uint16_t pc;
+
+		pc = dlb_read_pc(port_data, false);
+
+		qm_port->cached_dir_credits = pc -
+			qm_port->dir_pushcount_at_credit_expiry;
+
+		if (unlikely(qm_port->cached_dir_credits == 0)) {
+			DLB_INC_STAT(
+			qm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,
+			1);
+
+			DLB_LOG_DBG("dir credits exhausted\n");
+			return 1;
+		}
+		qm_port->dir_pushcount_at_credit_expiry +=
+			qm_port->cached_dir_credits;
+	}
+
+	return 0;
+}
+
+static inline int
+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,
+		       struct dlb_port *qm_port,
+		       const struct rte_event ev[],
+		       struct process_local_port_data *port_data,
+		       uint8_t *sched_type,
+		       uint8_t *queue_id)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	uint16_t *cached_credits = NULL;
+	struct dlb_queue *qm_queue;
+
+	ev_queue = &dlb->ev_queues[ev->queue_id];
+	qm_queue = &ev_queue->qm_queue;
+	*queue_id = qm_queue->id;
+
+	/* Ignore sched_type and hardware credits on release events */
+	if (ev->op == RTE_EVENT_OP_RELEASE)
+		goto op_check;
+
+	if (!qm_queue->is_directed) {
+		/* Load balanced destination queue */
+
+		if (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_ldb_credits;
+
+		switch (ev->sched_type) {
+		case RTE_SCHED_TYPE_ORDERED:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ORDERED\n");
+			if (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {
+				DLB_LOG_ERR("dlb: tried to send ordered event to unordered queue %d\n",
+					    *queue_id);
+				rte_errno = -EINVAL;
+				return 1;
+			}
+			*sched_type = DLB_SCHED_ORDERED;
+			break;
+		case RTE_SCHED_TYPE_ATOMIC:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\n");
+			*sched_type = DLB_SCHED_ATOMIC;
+			break;
+		case RTE_SCHED_TYPE_PARALLEL:
+			DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\n");
+			if (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)
+				*sched_type = DLB_SCHED_ORDERED;
+			else
+				*sched_type = DLB_SCHED_UNORDERED;
+			break;
+		default:
+			DLB_LOG_ERR("Unsupported LDB sched type in put_qe\n");
+			DLB_INC_STAT(ev_port->stats.tx_invalid, 1);
+			rte_errno = -EINVAL;
+			return 1;
+		}
+	} else {
+		/* Directed destination queue */
+
+		if (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		cached_credits = &qm_port->cached_dir_credits;
+
+		DLB_LOG_DBG("dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\n");
+
+		*sched_type = DLB_SCHED_DIRECTED;
+	}
+
+op_check:
+	switch (ev->op) {
+	case RTE_EVENT_OP_NEW:
+		/* Check that a sw credit is available */
+		if (dlb_check_enqueue_sw_credits(dlb, ev_port)) {
+			rte_errno = -ENOSPC;
+			return 1;
+		}
+		ev_port->inflight_credits--;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_FORWARD:
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		(*cached_credits)--;
+		break;
+	case RTE_EVENT_OP_RELEASE:
+		ev_port->inflight_credits++;
+		/* Check for outstanding_releases underflow. If this occurs,
+		 * the application is not using the EVENT_OPs correctly; for
+		 * example, forwarding or releasing events that were not
+		 * dequeued.
+		 */
+		RTE_ASSERT(ev_port->outstanding_releases > 0);
+		ev_port->outstanding_releases--;
+		qm_port->issued_releases++;
+		/* Replenish s/w credits if enough are cached */
+		dlb_replenish_sw_credits(dlb, ev_port);
+		break;
+	}
+
+	DLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);
+	DLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+	if (ev->op != RTE_EVENT_OP_RELEASE) {
+		DLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);
+		DLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);
+	}
+#endif
+
+	return 0;
+}
+
+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {
+	{
+		/* Load-balanced cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,
+	},
+	{
+		/* Directed cmd bytes */
+		[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,
+		[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,
+	},
+};
+
+static inline void
+dlb_event_build_hcws(struct dlb_port *qm_port,
+		     const struct rte_event ev[],
+		     int num,
+		     uint8_t *sched_type,
+		     uint8_t *queue_id)
+{
+	struct dlb_enqueue_qe *qe;
+	uint16_t sched_word[4];
+	__m128i sse_qe[2];
+	int i;
+
+	qe = qm_port->qe4;
+
+	sse_qe[0] = _mm_setzero_si128();
+	sse_qe[1] = _mm_setzero_si128();
+
+	switch (num) {
+	case 4:
+		/* Construct the metadata portion of two HCWs in one 128b SSE
+		 * register. HCW metadata is constructed in the SSE registers
+		 * like so:
+		 * sse_qe[0][63:0]:   qe[0]'s metadata
+		 * sse_qe[0][127:64]: qe[1]'s metadata
+		 * sse_qe[1][63:0]:   qe[2]'s metadata
+		 * sse_qe[1][127:64]: qe[3]'s metadata
+		 */
+
+		/* Convert the event operation into a command byte and store it
+		 * in the metadata:
+		 * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]
+		 * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]
+		 * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]
+		 * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]
+		 */
+#define DLB_QE_CMD_BYTE 7
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[0].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[0] = _mm_insert_epi8(sse_qe[0],
+				cmd_byte_map[qm_port->is_directed][ev[1].op],
+				DLB_QE_CMD_BYTE + 8);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[2].op],
+				DLB_QE_CMD_BYTE);
+		sse_qe[1] = _mm_insert_epi8(sse_qe[1],
+				cmd_byte_map[qm_port->is_directed][ev[3].op],
+				DLB_QE_CMD_BYTE + 8);
+
+		/* Store priority, scheduling type, and queue ID in the sched
+		 * word array because these values are re-used when the
+		 * destination is a directed queue.
+		 */
+		sched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |
+				sched_type[0] << 8 |
+				queue_id[0];
+		sched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |
+				sched_type[1] << 8 |
+				queue_id[1];
+		sched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |
+				sched_type[2] << 8 |
+				queue_id[2];
+		sched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |
+				sched_type[3] << 8 |
+				queue_id[3];
+
+		/* Store the event priority, scheduling type, and queue ID in
+		 * the metadata:
+		 * sse_qe[0][31:16] = sched_word[0]
+		 * sse_qe[0][95:80] = sched_word[1]
+		 * sse_qe[1][31:16] = sched_word[2]
+		 * sse_qe[1][95:80] = sched_word[3]
+		 */
+#define DLB_QE_QID_SCHED_WORD 1
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[0],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     sched_word[1],
+					     DLB_QE_QID_SCHED_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[2],
+					     DLB_QE_QID_SCHED_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     sched_word[3],
+					     DLB_QE_QID_SCHED_WORD + 4);
+
+		/* If the destination is a load-balanced queue, store the lock
+		 * ID. If it is a directed queue, DLB places this field in
+		 * bytes 10-11 of the received QE, so we format it accordingly:
+		 * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]
+		 * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]
+		 * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]
+		 * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]
+		 */
+#define DLB_QE_LOCK_ID_WORD 2
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[0] == DLB_SCHED_DIRECTED) ?
+					sched_word[0] : ev[0].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+				(sched_type[1] == DLB_SCHED_DIRECTED) ?
+					sched_word[1] : ev[1].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[2] == DLB_SCHED_DIRECTED) ?
+					sched_word[2] : ev[2].flow_id,
+				DLB_QE_LOCK_ID_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+				(sched_type[3] == DLB_SCHED_DIRECTED) ?
+					sched_word[3] : ev[3].flow_id,
+				DLB_QE_LOCK_ID_WORD + 4);
+
+		/* Store the event type and sub event type in the metadata:
+		 * sse_qe[0][15:0]  = flow_id[0]
+		 * sse_qe[0][79:64] = flow_id[1]
+		 * sse_qe[1][15:0]  = flow_id[2]
+		 * sse_qe[1][79:64] = flow_id[3]
+		 */
+#define DLB_QE_EV_TYPE_WORD 0
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[0].sub_event_type << 8 |
+						ev[0].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[0] = _mm_insert_epi16(sse_qe[0],
+					     ev[1].sub_event_type << 8 |
+						ev[1].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[2].sub_event_type << 8 |
+						ev[2].event_type,
+					     DLB_QE_EV_TYPE_WORD);
+		sse_qe[1] = _mm_insert_epi16(sse_qe[1],
+					     ev[3].sub_event_type << 8 |
+						ev[3].event_type,
+					     DLB_QE_EV_TYPE_WORD + 4);
+
+		/* Store the metadata to memory (use the double-precision
+		 * _mm_storeh_pd because there is no integer function for
+		 * storing the upper 64b):
+		 * qe[0] metadata = sse_qe[0][63:0]
+		 * qe[1] metadata = sse_qe[0][127:64]
+		 * qe[2] metadata = sse_qe[1][63:0]
+		 * qe[3] metadata = sse_qe[1][127:64]
+		 */
+		_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);
+		_mm_storeh_pd((double *)&qe[1].u.opaque_data,
+			      (__m128d) sse_qe[0]);
+		_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);
+		_mm_storeh_pd((double *)&qe[3].u.opaque_data,
+			      (__m128d) sse_qe[1]);
+
+		qe[0].data = ev[0].u64;
+		qe[1].data = ev[1].u64;
+		qe[2].data = ev[2].u64;
+		qe[3].data = ev[3].u64;
+
+		break;
+	case 3:
+	case 2:
+	case 1:
+		/* At least one QE will be valid, so only zero out three */
+		qe[1].cmd_byte = 0;
+		qe[2].cmd_byte = 0;
+		qe[3].cmd_byte = 0;
+
+		for (i = 0; i < num; i++) {
+			qe[i].cmd_byte =
+				cmd_byte_map[qm_port->is_directed][ev[i].op];
+			qe[i].sched_type = sched_type[i];
+			qe[i].data = ev[i].u64;
+			qe[i].qid = queue_id[i];
+			qe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);
+			qe[i].lock_id = ev[i].flow_id;
+			if (sched_type[i] == DLB_SCHED_DIRECTED) {
+				struct dlb_msg_info *info =
+					(struct dlb_msg_info *)&qe[i].lock_id;
+
+				info->qid = queue_id[i];
+				info->sched_type = DLB_SCHED_DIRECTED;
+				info->priority = qe[i].priority;
+			}
+			qe[i].u.event_type.major = ev[i].event_type;
+			qe[i].u.event_type.sub = ev[i].sub_event_type;
+		}
+		break;
+	}
+}
+
+static inline void
+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)
+{
+	struct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;
+	int num = qm_port->owed_tokens;
+
+	if (qm_port->use_rsvd_token_scheme) {
+		/* Check if there's a deficit of reserved tokens, and return
+		 * early if there are no (unreserved) tokens to consume.
+		 */
+		if (num <= qm_port->cq_rsvd_token_deficit) {
+			qm_port->cq_rsvd_token_deficit -= num;
+			qm_port->owed_tokens = 0;
+			return;
+		}
+		num -= qm_port->cq_rsvd_token_deficit;
+		qm_port->cq_rsvd_token_deficit = 0;
+	}
+
+	qe[idx].cmd_byte = DLB_POP_CMD_BYTE;
+	qe[idx].tokens = num - 1;
+
+	qm_port->owed_tokens = 0;
+}
+
+static inline uint16_t
+__dlb_event_enqueue_burst(void *event_port,
+			  const struct rte_event events[],
+			  uint16_t num,
+			  bool use_delayed)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct process_local_port_data *port_data;
+	int i, cnt;
+
+	RTE_ASSERT(ev_port->enq_configured);
+	RTE_ASSERT(events != NULL);
+
+	rte_errno = 0;
+	cnt = 0;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	if (!port_data->mmaped)
+		dlb_iface_port_mmap(qm_port);
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	rte_wmb();
+
+	for (i = 0; i < num; i += DLB_NUM_QES_PER_CACHE_LINE) {
+		uint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];
+		int j = 0;
+
+		for (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {
+			const struct rte_event *ev = &events[i + j];
+
+			if (dlb_event_enqueue_prep(ev_port, qm_port, ev,
+						   port_data, &sched_types[j],
+						   &queue_ids[j]))
+				break;
+		}
+
+		if (j == 0)
+			break;
+
+		dlb_event_build_hcws(qm_port, &events[i], j,
+				     sched_types, queue_ids);
+
+		/* The delayed-pop code causes an unnecessary performance
+		 * penalty when it is not in use. The use_delayed argument
+		 * allows the compiler to create a version of this function
+		 * with these checks factored out that the PMD can call
+		 * when delayed-pop is not in use.
+		 */
+		if (use_delayed &&
+		    qm_port->token_pop_mode == DELAYED_POP && j < 4 &&
+		    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
+
+			dlb_construct_token_pop_qe(qm_port, j);
+
+			/* Reset the releases counter for the next QE batch */
+			qm_port->issued_releases -= qm_port->token_pop_thresh;
+
+			/* When using delayed token pop mode, the initial token
+			 * threshold is the full CQ depth. After the first
+			 * token pop, we need to reset it to the dequeue_depth.
+			 */
+			qm_port->token_pop_thresh = qm_port->dequeue_depth;
+		}
+
+		dlb_hw_do_enqueue(qm_port, port_data);
+
+		cnt += j;
+
+		if (j < DLB_NUM_QES_PER_CACHE_LINE)
+			break;
+	}
+
+	if (use_delayed && qm_port->token_pop_mode == DELAYED_POP &&
+	    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+		qm_port->issued_releases -= qm_port->token_pop_thresh;
+		qm_port->token_pop_thresh = qm_port->dequeue_depth;
+	}
+
+	RTE_ASSERT(!((cnt == 0 && rte_errno != -ENOSPC)));
+
+	return cnt;
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst(void *event_port,
+			 const struct rte_event events[],
+			 uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_burst_delayed(void *event_port,
+				 const struct rte_event events[],
+				 uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
+}
+
+static inline uint16_t
+dlb_event_enqueue(void *event_port,
+		   const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1, false);
+}
+
+static inline uint16_t
+dlb_event_enqueue_delayed(void *event_port,
+			   const struct rte_event events[])
+{
+	return __dlb_event_enqueue_burst(event_port, events, 1, true);
+}
+
+static uint16_t
+dlb_event_enqueue_new_burst_delayed(void *event_port,
+				     const struct rte_event events[],
+				     uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
+}
+
+static uint16_t
+dlb_event_enqueue_forward_burst_delayed(void *event_port,
+					 const struct rte_event events[],
+					 uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, true);
+}
+
 /* Note: 1 QM instance per QM device, QM instance/device == event device */
 static int
 dlb_eventdev_configure(const struct rte_eventdev *dev)
@@ -935,6 +2048,133 @@ dlb_eventdev_queue_setup(struct rte_eventdev *dev,
 	return ret;
 }
 
+static void
+dlb_port_link_teardown(struct dlb_eventdev *dlb,
+		       struct dlb_eventdev_port *ev_port)
+{
+	struct dlb_eventdev_queue *ev_queue;
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (!ev_port->link[i].valid)
+			continue;
+
+		ev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];
+
+		ev_port->link[i].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+}
+
+static int
+dlb_eventdev_port_setup(struct rte_eventdev *dev,
+			uint8_t ev_port_id,
+			const struct rte_event_port_conf *port_conf)
+{
+	struct dlb_eventdev *dlb;
+	struct dlb_eventdev_port *ev_port;
+	bool use_rsvd_token_scheme;
+	uint32_t adj_cq_depth;
+	uint16_t rsvd_tokens;
+	int ret;
+
+	if (dev == NULL || port_conf == NULL) {
+		DLB_LOG_ERR("Null parameter\n");
+		return -EINVAL;
+	}
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (ev_port_id >= DLB_MAX_NUM_PORTS)
+		return -EINVAL;
+
+	if (port_conf->dequeue_depth >
+		evdev_dlb_default_info.max_event_port_dequeue_depth ||
+	    port_conf->enqueue_depth >
+		evdev_dlb_default_info.max_event_port_enqueue_depth)
+		return -EINVAL;
+
+	ev_port = &dlb->ev_ports[ev_port_id];
+	/* configured? */
+	if (ev_port->setup_done) {
+		DLB_LOG_ERR("evport %d is already configured\n", ev_port_id);
+		return -EINVAL;
+	}
+
+	/* The reserved token interrupt arming scheme requires that one or more
+	 * CQ tokens be reserved by the PMD. This limits the amount of CQ space
+	 * usable by the DLB, so in order to give an *effective* CQ depth equal
+	 * to the user-requested value, we double CQ depth and reserve half of
+	 * its tokens. If the user requests the max CQ depth (256) then we
+	 * cannot double it, so we reserve one token and give an effective
+	 * depth of 255 entries.
+	 */
+	use_rsvd_token_scheme = true;
+	rsvd_tokens = 1;
+	adj_cq_depth = port_conf->dequeue_depth;
+
+	if (use_rsvd_token_scheme && adj_cq_depth < 256) {
+		rsvd_tokens = adj_cq_depth;
+		adj_cq_depth *= 2;
+	}
+
+	ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+		RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+	if (!ev_port->qm_port.is_directed) {
+		ret = dlb_hw_create_ldb_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+				    ev_port_id);
+			return ret;
+		}
+	} else {
+		ret = dlb_hw_create_dir_port(dlb,
+					     ev_port,
+					     port_conf->dequeue_depth,
+					     adj_cq_depth,
+					     port_conf->enqueue_depth,
+					     rsvd_tokens,
+					     use_rsvd_token_scheme);
+		if (ret < 0) {
+			DLB_LOG_ERR("Failed to create the DIR port\n");
+			return ret;
+		}
+	}
+
+	/* Save off port config for reconfig */
+	dlb->ev_ports[ev_port_id].conf = *port_conf;
+
+	dlb->ev_ports[ev_port_id].id = ev_port_id;
+	dlb->ev_ports[ev_port_id].enq_configured = true;
+	dlb->ev_ports[ev_port_id].setup_done = true;
+	dlb->ev_ports[ev_port_id].inflight_max =
+		port_conf->new_event_threshold;
+	dlb->ev_ports[ev_port_id].implicit_release =
+		!(port_conf->event_port_cfg &
+		  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+	dlb->ev_ports[ev_port_id].outstanding_releases = 0;
+	dlb->ev_ports[ev_port_id].inflight_credits = 0;
+	dlb->ev_ports[ev_port_id].credit_update_quanta =
+		RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;
+	dlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */
+
+	/* Tear down pre-existing port->queue links */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		dlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);
+
+	dev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];
+
+	return 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -1014,6 +2254,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
+		.port_setup       = dlb_eventdev_port_setup,
 	};
 
 	/* Expose PMD's eventdev interface */
-- 
2.13.6

[dpdk-dev] [PATCH 20/27] event/dlb: add port_link

[McDaniel, Timothy]

Change-Id: Id86dbdd1070e8102bf602765654fe5afde0e0d6c
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 303 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 303 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index deea474bf..7acf8b33d 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -794,6 +794,29 @@ dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,
 	return qm_qid;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
 static inline void
 dlb_hw_do_enqueue(struct dlb_port *qm_port,
 		  struct process_local_port_data *port_data)
@@ -803,6 +826,42 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,
 	dlb_pp_write(qm_port->qe4, port_data);
 }
 
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
 /* VDEV-only notes:
  * This function first unmaps all memory mappings and closes the
  * domain's file descriptor, which causes the driver to reset the
@@ -1785,6 +1844,249 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -2255,6 +2557,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 	};
 
 	/* Expose PMD's eventdev interface */
-- 
2.13.6

[dpdk-dev] [PATCH 21/27] event/dlb: add queue_release and port_release

[McDaniel, Timothy]

Change-Id: I7dc1b4c03a222b40d629705946d00f3800dbe63d
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 7acf8b33d..b1103ea95 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -2477,6 +2477,29 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	RTE_SET_USED(port);
+
+	/* This function intentionally left blank. dlb does not support
+	 * reconfiguring individual queues or ports -- the entire device
+	 * must be reconfigured.
+	 */
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. dlb does not support
+	 * reconfiguring individual queues or ports -- the entire device
+	 * must be reconfigured.
+	 */
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -2555,8 +2578,10 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_configure    = dlb_eventdev_configure,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 	};
 
-- 
2.13.6

[dpdk-dev] [PATCH 22/27] event/dlb: add port_unlink and port_unlinks_in_progress

[McDaniel, Timothy]

Change-Id: I06d4006786440a0454ce883dba0ce14640dfff92
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index b1103ea95..2d7f4479a 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -862,6 +862,30 @@ dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 	return ret;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
 /* VDEV-only notes:
  * This function first unmaps all memory mappings and closes the
  * domain's file descriptor, which causes the driver to reset the
@@ -1905,6 +1929,42 @@ dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
 }
 
 static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attemmpts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
 		 struct dlb_eventdev_port *ev_port,
@@ -2477,6 +2537,109 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 	return 0;
 }
 
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!queues || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_release(void *port)
 {
@@ -2583,6 +2746,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 	};
 
 	/* Expose PMD's eventdev interface */
-- 
2.13.6

[dpdk-dev] [PATCH 23/27] event/dlb: add eventdev_start

[McDaniel, Timothy]

Change-Id: I9dcf8cf95c1001e09ab3e44e3fbd2ccbf0ddec72
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 142 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 142 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 2d7f4479a..70bd2b4d6 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1162,6 +1162,72 @@ dlb_eventdev_info_get(struct rte_eventdev *dev,
 	*dev_info = evdev_dlb_default_info;
 }
 
+static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev);
+
+static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev);
+
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 static inline int
 dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
 			     struct dlb_eventdev_port *ev_port)
@@ -2058,6 +2124,40 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2538,6 +2638,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
 			 uint8_t queues[], uint16_t nb_unlinks)
 {
@@ -2739,6 +2880,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.queue_release    = dlb_eventdev_queue_release,
-- 
2.13.6

[dpdk-dev] [PATCH 24/27] event/dlb: add timout_ticks, dump, xstats, and selftest

[McDaniel, Timothy]

Change-Id: I07517f651b7cf2326b6d3c62f0089595c9eb06a7
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 70bd2b4d6..eef7c4f43 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1820,6 +1820,20 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 	return __dlb_event_enqueue_burst(event_port, events, num, true);
 }
 
+
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns;
+
+	cycles_per_ns = rte_get_timer_hz() / 1E9;
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 /* Note: 1 QM instance per QM device, QM instance/device == event device */
 static int
 dlb_eventdev_configure(const struct rte_eventdev *dev)
@@ -2891,6 +2905,13 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
-- 
2.13.6

[dpdk-dev] [PATCH 25/27] event/dlb: add enqueue and its burst variants

[McDaniel, Timothy]

Change-Id: I58cb02dc040aa6dcf30992ea57ff3f332b4946f2
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index eef7c4f43..282d60f73 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -1805,6 +1805,14 @@ dlb_event_enqueue_delayed(void *event_port,
 }
 
 static uint16_t
+dlb_event_enqueue_new_burst(void *event_port,
+			     const struct rte_event events[],
+			     uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
 dlb_event_enqueue_new_burst_delayed(void *event_port,
 				     const struct rte_event events[],
 				     uint16_t num)
@@ -1813,6 +1821,14 @@ dlb_event_enqueue_new_burst_delayed(void *event_port,
 }
 
 static uint16_t
+dlb_event_enqueue_forward_burst(void *event_port,
+				 const struct rte_event events[],
+				 uint16_t num)
+{
+	return __dlb_event_enqueue_burst(event_port, events, num, false);
+}
+
+static uint16_t
 dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					 const struct rte_event events[],
 					 uint16_t num)
@@ -2917,6 +2933,10 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	/* Expose PMD's eventdev interface */
 
 	dev->dev_ops = &dlb_eventdev_entry_ops;
+	dev->enqueue = dlb_event_enqueue;
+	dev->enqueue_burst = dlb_event_enqueue_burst;
+	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
+	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
 
 }
 
-- 
2.13.6

[dpdk-dev] [PATCH 26/27] event/dlb: add dequeue, dequeue_burst, and variants

[McDaniel, Timothy]

Change-Id: I0c90ad10be82750dc4c7c4bcbeed9b12677a996d
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 879 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 879 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index 282d60f73..afdb18061 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -826,6 +826,700 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,
 	dlb_pp_write(qm_port->qe4, port_data);
 }
 
+static __rte_always_inline int
+dlb_recv_qe(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe,
+	    uint8_t *offset)
+{
+	uint8_t xor_mask[2][4] = { {0x0F, 0x0E, 0x0C, 0x08},
+				   {0x00, 0x01, 0x03, 0x07} };
+	uint8_t and_mask[4] = {0x0F, 0x0E, 0x0C, 0x08};
+	volatile struct dlb_dequeue_qe *cq_addr;
+	__m128i *qes = (__m128i *)qe;
+	uint64_t *cache_line_base;
+	uint8_t gen_bits;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	cq_addr = &cq_addr[qm_port->cq_idx];
+
+	cache_line_base = (void *)(((uintptr_t)cq_addr) & ~0x3F);
+	*offset = ((uintptr_t)cq_addr & 0x30) >> 4;
+
+	/* Load the next CQ cache line from memory. Pack these reads as tight
+	 * as possible to reduce the chance that DLB invalidates the line while
+	 * the CPU is reading it. Read the cache line backwards to ensure that
+	 * if QE[N] (N > 0) is valid, then QEs[0:N-1] are too.
+	 *
+	 * (Valid QEs start at &qe[offset])
+	 */
+	qes[3] = _mm_load_si128((__m128i *)&cache_line_base[6]);
+	qes[2] = _mm_load_si128((__m128i *)&cache_line_base[4]);
+	qes[1] = _mm_load_si128((__m128i *)&cache_line_base[2]);
+	qes[0] = _mm_load_si128((__m128i *)&cache_line_base[0]);
+
+	/* Evict the cache line ASAP */
+	dlb_cldemote(cache_line_base);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[qm_port->gen_bit][*offset];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask[*offset];
+
+	return __builtin_popcount(gen_bits);
+}
+
+static __rte_always_inline int
+dlb_recv_qe_sparse(struct dlb_port *qm_port, struct dlb_dequeue_qe *qe)
+{
+	volatile struct dlb_dequeue_qe *cq_addr;
+	uint8_t xor_mask[2] = {0x0F, 0x00};
+	const uint8_t and_mask = 0x0F;
+	__m128i *qes = (__m128i *)qe;
+	uint8_t gen_bits, gen_bit;
+	uintptr_t addr[4];
+	uint16_t idx;
+
+	cq_addr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+
+	idx = qm_port->cq_idx;
+
+	/* Load the next 4 QEs */
+	addr[0] = (uintptr_t)&cq_addr[idx];
+	addr[1] = (uintptr_t)&cq_addr[(idx +  4) & qm_port->cq_depth_mask];
+	addr[2] = (uintptr_t)&cq_addr[(idx +  8) & qm_port->cq_depth_mask];
+	addr[3] = (uintptr_t)&cq_addr[(idx + 12) & qm_port->cq_depth_mask];
+
+	/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */
+	rte_prefetch0(&cq_addr[(idx + 16) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 20) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 24) & qm_port->cq_depth_mask]);
+	rte_prefetch0(&cq_addr[(idx + 28) & qm_port->cq_depth_mask]);
+
+	/* Correct the xor_mask for wrap-around QEs */
+	gen_bit = qm_port->gen_bit;
+	xor_mask[gen_bit] ^= !!((idx +  4) > qm_port->cq_depth_mask) << 1;
+	xor_mask[gen_bit] ^= !!((idx +  8) > qm_port->cq_depth_mask) << 2;
+	xor_mask[gen_bit] ^= !!((idx + 12) > qm_port->cq_depth_mask) << 3;
+
+	/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is
+	 * valid, then QEs[0:N-1] are too.
+	 */
+	qes[3] = _mm_load_si128((__m128i *)(void *)addr[3]);
+	rte_compiler_barrier();
+	qes[2] = _mm_load_si128((__m128i *)(void *)addr[2]);
+	rte_compiler_barrier();
+	qes[1] = _mm_load_si128((__m128i *)(void *)addr[1]);
+	rte_compiler_barrier();
+	qes[0] = _mm_load_si128((__m128i *)(void *)addr[0]);
+
+	/* Extract and combine the gen bits */
+	gen_bits = ((_mm_extract_epi8(qes[0], 15) & 0x1) << 0) |
+		   ((_mm_extract_epi8(qes[1], 15) & 0x1) << 1) |
+		   ((_mm_extract_epi8(qes[2], 15) & 0x1) << 2) |
+		   ((_mm_extract_epi8(qes[3], 15) & 0x1) << 3);
+
+	/* XOR the combined bits such that a 1 represents a valid QE */
+	gen_bits ^= xor_mask[gen_bit];
+
+	/* Mask off gen bits we don't care about */
+	gen_bits &= and_mask;
+
+	return __builtin_popcount(gen_bits);
+}
+
+static inline void
+dlb_inc_cq_idx(struct dlb_port *qm_port, int cnt)
+{
+	uint16_t idx = qm_port->cq_idx_unmasked + cnt;
+
+	qm_port->cq_idx_unmasked = idx;
+	qm_port->cq_idx = idx & qm_port->cq_depth_mask;
+	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;
+}
+
+static inline bool
+dlb_cq_is_empty(struct dlb_port *qm_port)
+{
+	volatile struct dlb_dequeue_qe *qe_ptr;
+	struct dlb_dequeue_qe qe;
+
+	qe_ptr = dlb_port[qm_port->id][PORT_TYPE(qm_port)].cq_base;
+	qe = qe_ptr[qm_port->cq_idx];
+
+	return (qe.cq_gen != qm_port->gen_bit);
+}
+
+static inline void dlb_send_rsvd_token_int_arm(struct dlb_port *qm_port)
+{
+	struct dlb_cq_pop_qe *qe = qm_port->consume_qe;
+	struct process_local_port_data *port_data;
+
+	qe->tokens = 0;
+
+	rte_memcpy(&qm_port->qe4[0], qe, sizeof(struct dlb_cq_pop_qe));
+
+	/* Set int_arm to true in qm_port->qe4[0] so consume_qe isn't
+	 * affected.
+	 */
+	qm_port->qe4[0].int_arm = 1;
+
+	/* Fill remaining 3 entries (1,2, and 3) with noop QEs */
+
+	memset(&qm_port->qe4[1],
+	       0,
+	       (DLB_NUM_QES_PER_CACHE_LINE - 1) *
+		sizeof(struct dlb_cq_pop_qe));
+
+	/* No store fence needed since no pointer is being sent, and CQ token
+	 * pops can be safely reordered with other HCWs.
+	 */
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	dlb_pp_write(qm_port->qe4, port_data);
+
+	DLB_LOG_DBG("dlb: send rsvd token int arm\n");
+}
+
+static inline void dlb_issue_int_arm_hcw(struct dlb_port *qm_port)
+{
+	dlb_send_rsvd_token_int_arm(qm_port);
+
+	qm_port->cq_rsvd_token_deficit = 1;
+	qm_port->int_armed = true;
+
+	/* The delayed token return needs to account for this token pop HCW */
+	qm_port->issued_releases--;
+}
+
+static inline int
+dlb_block_on_cq_interrupt(struct dlb_hw_dev *handle,
+			  struct dlb_port *qm_port,
+			  struct dlb_eventdev_port *ev_port)
+{
+	struct process_local_port_data *port_data;
+	int ret;
+
+	/* If the interrupt is not armed, either sleep-poll (see comment below)
+	 * or arm the interrupt.
+	 */
+	if (!qm_port->int_armed) {
+		/* The reserved token scheme requires setting the interrupt
+		 * depth threshold equal to the number of reserved tokens.
+		 * Until the port receives its set of reserved tokens it cannot
+		 * block on the interrupt, since it will not fire until the CQ
+		 * contains at least num-reserved-tokens QEs. In lieu of that,
+		 * we sleep-poll the CQ, which gives a similar behavior (other
+		 * threads can run) albeit with a potentially slower response
+		 * time.
+		 */
+		if (qm_port->use_rsvd_token_scheme &&
+		    qm_port->cq_rsvd_token_deficit) {
+			sched_yield();
+			return 0;
+		}
+
+		dlb_issue_int_arm_hcw(qm_port);
+	}
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	/* Note: it's safe to access the per-process cq_base address here,
+	 * since the PMD won't block on the CQ until after attempting at least
+	 * one CQ dequeue.
+	 */
+	ret = dlb_iface_block_on_cq_interrupt(
+					handle,
+					qm_port->id,
+					!ev_port->qm_port.is_directed,
+					&port_data->cq_base[qm_port->cq_idx],
+					qm_port->gen_bit,
+					false);
+
+	/* If the CQ int ioctl was unsuccessful, the interrupt remains armed */
+	qm_port->int_armed = (ret != 0);
+
+	return ret;
+}
+
+static inline int
+dlb_process_dequeue_qes(struct dlb_eventdev_port *ev_port,
+			struct dlb_port *qm_port,
+			struct rte_event *events,
+			struct dlb_dequeue_qe *qes,
+			int cnt)
+{
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	int i, num;
+
+	RTE_SET_USED(ev_port);  /* avoids unused variable error */
+
+	for (i = 0, num = 0; i < cnt; i++) {
+		struct dlb_dequeue_qe *qe = &qes[i];
+		int sched_type_map[4] = {
+			[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+			[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+			[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+			[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+		};
+
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qe->data, qe->qid,
+			    qe->u.event_type.major,
+			    qe->u.event_type.sub,
+			    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+		/* Fill in event information.
+		 * Note that flow_id must be embedded in the data by
+		 * the app, such as the mbuf RSS hash field if the data
+		 * buffer is a mbuf.
+		 */
+		if (unlikely(qe->error)) {
+			DLB_LOG_ERR("QE error bit ON\n");
+			DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+			dlb_consume_qe_immediate(qm_port, 1);
+			continue; /* Ignore */
+		}
+
+		events[num].u64 = qe->data;
+		events[num].queue_id = qid_mappings[qe->qid];
+		events[num].priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+		events[num].event_type = qe->u.event_type.major;
+		events[num].sub_event_type = qe->u.event_type.sub;
+		events[num].sched_type = sched_type_map[qe->sched_type];
+		DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+
+		DLB_INC_STAT(ev_port->stats.traffic.rx_ok, 1);
+
+		num++;
+	}
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_four_qes(struct dlb_eventdev_port *ev_port,
+			     struct dlb_port *qm_port,
+			     struct rte_event *events,
+			     struct dlb_dequeue_qe *qes)
+{
+	int sched_type_map[] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	const int num_events = DLB_NUM_QES_PER_CACHE_LINE;
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+	__m128i sse_evt[2];
+#ifdef DLB_QUELL_STATS
+	(void)ev_port;  /* avoids unused variable error */
+#endif
+	int i;
+
+	/* In the unlikely case that any of the QE error bits are set, process
+	 * them one at a time.
+	 */
+	if (unlikely(qes[0].error || qes[1].error ||
+		     qes[2].error || qes[3].error))
+		return dlb_process_dequeue_qes(ev_port, qm_port, events,
+					       qes, num_events);
+
+	for (i = 0; i < DLB_NUM_QES_PER_CACHE_LINE; i++) {
+		DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+			    (long long)qes[i].data, qes[i].qid,
+			    qes[i].u.event_type.major,
+			    qes[i].u.event_type.sub,
+			    qes[i].pp_id, qes[i].sched_type, qes[i].qid,
+			    qes[i].error);
+	}
+
+	events[0].u64 = qes[0].data;
+	events[1].u64 = qes[1].data;
+	events[2].u64 = qes[2].data;
+	events[3].u64 = qes[3].data;
+
+	/* Construct the metadata portion of two struct rte_events
+	 * in one 128b SSE register. Event metadata is constructed in the SSE
+	 * registers like so:
+	 * sse_evt[0][63:0]:   event[0]'s metadata
+	 * sse_evt[0][127:64]: event[1]'s metadata
+	 * sse_evt[1][63:0]:   event[2]'s metadata
+	 * sse_evt[1][127:64]: event[3]'s metadata
+	 */
+	sse_evt[0] = _mm_setzero_si128();
+	sse_evt[1] = _mm_setzero_si128();
+
+	/* Convert the hardware queue ID to an event queue ID and store it in
+	 * the metadata:
+	 * sse_evt[0][47:40]   = qid_mappings[qes[0].qid]
+	 * sse_evt[0][111:104] = qid_mappings[qes[1].qid]
+	 * sse_evt[1][47:40]   = qid_mappings[qes[2].qid]
+	 * sse_evt[1][111:104] = qid_mappings[qes[3].qid]
+	 */
+#define RTE_EVENT_QUEUE_ID_BYTE 5
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[0].qid],
+				     RTE_EVENT_QUEUE_ID_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     qid_mappings[qes[1].qid],
+				     RTE_EVENT_QUEUE_ID_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[2].qid],
+				     RTE_EVENT_QUEUE_ID_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     qid_mappings[qes[3].qid],
+				     RTE_EVENT_QUEUE_ID_BYTE + 8);
+
+	/* Convert the hardware priority to an event priority and store it in
+	 * the metadata:
+	 * sse_evt[0][55:48]   = DLB_TO_EV_PRIO(qes[0].priority)
+	 * sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority)
+	 * sse_evt[1][55:48]   = DLB_TO_EV_PRIO(qes[2].priority)
+	 * sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority)
+	 */
+#define RTE_EVENT_PRIO_BYTE 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[0].priority),
+				     RTE_EVENT_PRIO_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+				     DLB_TO_EV_PRIO((uint8_t)qes[1].priority),
+				     RTE_EVENT_PRIO_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[2].priority),
+				     RTE_EVENT_PRIO_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+				     DLB_TO_EV_PRIO((uint8_t)qes[3].priority),
+				     RTE_EVENT_PRIO_BYTE + 8);
+
+	/* Write the event type and sub event type to the event metadata. Leave
+	 * flow ID unspecified, since the hardware does not maintain it during
+	 * scheduling:
+	 * sse_evt[0][31:0]   = qes[0].u.event_type.major << 28 |
+	 *			qes[0].u.event_type.sub << 20;
+	 * sse_evt[0][95:64]  = qes[1].u.event_type.major << 28 |
+	 *			qes[1].u.event_type.sub << 20;
+	 * sse_evt[1][31:0]   = qes[2].u.event_type.major << 28 |
+	 *			qes[2].u.event_type.sub << 20;
+	 * sse_evt[1][95:64]  = qes[3].u.event_type.major << 28 |
+	 *			qes[3].u.event_type.sub << 20;
+	 */
+#define RTE_EVENT_EV_TYPE_DW 0
+#define RTE_EVENT_EV_TYPE_SHIFT 28
+#define RTE_EVENT_SUB_EV_TYPE_SHIFT 20
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[0].u.event_type.major << RTE_EVENT_EV_TYPE_SHIFT |
+			qes[0].u.event_type.sub << RTE_EVENT_SUB_EV_TYPE_SHIFT,
+			RTE_EVENT_EV_TYPE_DW);
+	sse_evt[0] = _mm_insert_epi32(sse_evt[0],
+			qes[1].u.event_type.major << RTE_EVENT_EV_TYPE_SHIFT |
+			qes[1].u.event_type.sub <<  RTE_EVENT_SUB_EV_TYPE_SHIFT,
+			RTE_EVENT_EV_TYPE_DW + 2);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[2].u.event_type.major << RTE_EVENT_EV_TYPE_SHIFT |
+			qes[2].u.event_type.sub <<  RTE_EVENT_SUB_EV_TYPE_SHIFT,
+			RTE_EVENT_EV_TYPE_DW);
+	sse_evt[1] = _mm_insert_epi32(sse_evt[1],
+			qes[3].u.event_type.major << RTE_EVENT_EV_TYPE_SHIFT  |
+			qes[3].u.event_type.sub << RTE_EVENT_SUB_EV_TYPE_SHIFT,
+			RTE_EVENT_EV_TYPE_DW + 2);
+
+	/* Write the sched type to the event metadata. 'op' and 'rsvd' are not
+	 * set:
+	 * sse_evt[0][39:32]  = sched_type_map[qes[0].sched_type] << 6
+	 * sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6
+	 * sse_evt[1][39:32]  = sched_type_map[qes[2].sched_type] << 6
+	 * sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6
+	 */
+#define RTE_EVENT_SCHED_TYPE_BYTE 4
+#define RTE_EVENT_SCHED_TYPE_SHIFT 6
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[0].sched_type] << RTE_EVENT_SCHED_TYPE_SHIFT,
+		RTE_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[0] = _mm_insert_epi8(sse_evt[0],
+		sched_type_map[qes[1].sched_type] << RTE_EVENT_SCHED_TYPE_SHIFT,
+		RTE_EVENT_SCHED_TYPE_BYTE + 8);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[2].sched_type] << RTE_EVENT_SCHED_TYPE_SHIFT,
+		RTE_EVENT_SCHED_TYPE_BYTE);
+	sse_evt[1] = _mm_insert_epi8(sse_evt[1],
+		sched_type_map[qes[3].sched_type] << RTE_EVENT_SCHED_TYPE_SHIFT,
+		RTE_EVENT_SCHED_TYPE_BYTE + 8);
+
+	/* Store the metadata to the event (use the double-precision
+	 * _mm_storeh_pd because there is no integer function for storing the
+	 * upper 64b):
+	 * events[0].event = sse_evt[0][63:0]
+	 * events[1].event = sse_evt[0][127:64]
+	 * events[2].event = sse_evt[1][63:0]
+	 * events[3].event = sse_evt[1][127:64]
+	 */
+	_mm_storel_epi64((__m128i *)&events[0].event, sse_evt[0]);
+	_mm_storeh_pd((double *)&events[1].event, (__m128d) sse_evt[0]);
+	_mm_storel_epi64((__m128i *)&events[2].event, sse_evt[1]);
+	_mm_storeh_pd((double *)&events[3].event, (__m128d) sse_evt[1]);
+
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[0].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[1].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[2].sched_type], 1);
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qes[3].sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, num_events);
+
+	return num_events;
+}
+
+static inline int
+dlb_dequeue_wait(struct dlb_eventdev *dlb,
+		 struct dlb_eventdev_port *ev_port,
+		 struct dlb_port *qm_port,
+		 uint64_t timeout,
+		 uint64_t start_ticks)
+{
+	struct process_local_port_data *port_data;
+	uint64_t elapsed_ticks;
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	elapsed_ticks = rte_get_timer_cycles() - start_ticks;
+
+	/* Wait/poll time expired */
+	if (elapsed_ticks >= timeout) {
+		/* Interrupts not supported by PF PMD */
+		return 1;
+	} else if (dlb->umwait_allowed) {
+		volatile struct dlb_dequeue_qe *cq_base;
+
+		cq_base = port_data->cq_base;
+
+		/* Block on cache line write to CQ. Note: it's
+		 * safe to access the per-process cq_base
+		 * address here, since the PMD has already
+		 * attempted at least one CQ dequeue.
+		 */
+		dlb_umonitor(&cq_base[qm_port->cq_idx]);
+
+		/* Avoid race condition. Check if still empty */
+		if (dlb_cq_is_empty(qm_port)) {
+			dlb_umwait(RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE,
+				   timeout + start_ticks);
+			DLB_INC_STAT(ev_port->stats.traffic.rx_umonitor_umwait,
+				     1);
+		}
+	} else {
+		uint64_t poll_interval = RTE_LIBRTE_PMD_DLB_POLL_INTERVAL;
+		uint64_t curr_ticks = rte_get_timer_cycles();
+		uint64_t init_ticks = curr_ticks;
+
+		while ((curr_ticks - start_ticks < timeout) &&
+		       (curr_ticks - init_ticks < poll_interval))
+			curr_ticks = rte_get_timer_cycles();
+	}
+
+	return 0;
+}
+
+static inline int16_t
+dlb_hw_dequeue(struct dlb_eventdev *dlb,
+	       struct dlb_eventdev_port *ev_port,
+	       struct rte_event *events,
+	       uint16_t max_num,
+	       uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* We have a special implementation for waiting. Wait can be:
+	 * 1) no waiting at all
+	 * 2) busy poll only
+	 * 3) wait for interrupt. If wakeup and poll time
+	 * has expired, then return to caller
+	 * 4) umonitor/umwait repeatedly up to poll time
+	 */
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		uint8_t offset;
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe(qm_port, qes, &offset);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[offset]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[offset],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
+static inline int
+dlb_process_dequeue_qe(struct dlb_eventdev_port *ev_port __rte_unused,
+		       struct dlb_port *qm_port,
+		       struct rte_event *event,
+		       struct dlb_dequeue_qe *qe)
+{
+	int sched_type_map[4] = {
+		[DLB_SCHED_ATOMIC] = RTE_SCHED_TYPE_ATOMIC,
+		[DLB_SCHED_UNORDERED] = RTE_SCHED_TYPE_PARALLEL,
+		[DLB_SCHED_ORDERED] = RTE_SCHED_TYPE_ORDERED,
+		[DLB_SCHED_DIRECTED] = RTE_SCHED_TYPE_ATOMIC,
+	};
+	uint8_t *qid_mappings = qm_port->qid_mappings;
+
+	DLB_LOG_DBG("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n",
+		    (long long)qe->data, qe->qid,
+		    qe->u.event_type.major,
+		    qe->u.event_type.sub,
+		    qe->pp_id, qe->sched_type, qe->qid, qe->error);
+
+	/* Fill in event information.
+	 * Note that flow_id must be embedded in the data by
+	 * the app, such as the mbuf RSS hash field if the data
+	 * buffer is a mbuf.
+	 */
+	if (unlikely(qe->error)) {
+		DLB_LOG_ERR("QE error bit ON\n");
+		DLB_INC_STAT(ev_port->stats.traffic.rx_drop, 1);
+		dlb_consume_qe_immediate(qm_port, 1);
+		return 0;
+	}
+
+	event->u64 = qe->data;
+	event->queue_id = qid_mappings[qe->qid];
+	event->priority = DLB_TO_EV_PRIO((uint8_t)qe->priority);
+	event->event_type = qe->u.event_type.major;
+	event->sub_event_type = qe->u.event_type.sub;
+	event->sched_type = sched_type_map[qe->sched_type];
+	DLB_INC_STAT(ev_port->stats.rx_sched_cnt[qe->sched_type], 1);
+
+	DLB_INC_STAT(ev_port->stats.traffic.rx_ok, 1);
+
+	return 1;
+}
+
+static inline int16_t
+dlb_hw_dequeue_sparse(struct dlb_eventdev *dlb,
+		      struct dlb_eventdev_port *ev_port,
+		      struct rte_event *events,
+		      uint16_t max_num,
+		      uint64_t dequeue_timeout_ticks)
+{
+	uint64_t timeout;
+	uint64_t start_ticks = 0ULL;
+	struct dlb_port *qm_port;
+	int num = 0;
+
+	qm_port = &ev_port->qm_port;
+
+	/* We have a special implementation for waiting. Wait can be:
+	 * 1) no waiting at all
+	 * 2) busy poll only
+	 * 3) wait for interrupt. If wakeup and poll time
+	 * has expired, then return to caller
+	 * 4) umonitor/umwait repeatedly up to poll time
+	 */
+
+	/* If configured for per dequeue wait, then use wait value provided
+	 * to this API. Otherwise we must use the global
+	 * value from eventdev config time.
+	 */
+	if (!dlb->global_dequeue_wait)
+		timeout = dequeue_timeout_ticks;
+	else
+		timeout = dlb->global_dequeue_wait_ticks;
+
+	if (timeout)
+		start_ticks = rte_get_timer_cycles();
+
+	while (num < max_num) {
+		struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE];
+		int num_avail;
+
+		/* Copy up to 4 QEs from the current cache line into qes */
+		num_avail = dlb_recv_qe_sparse(qm_port, qes);
+
+		/* But don't process more than the user requested */
+		num_avail = RTE_MIN(num_avail, max_num - num);
+
+		dlb_inc_cq_idx(qm_port, num_avail << 2);
+
+		if (num_avail == DLB_NUM_QES_PER_CACHE_LINE)
+			num += dlb_process_dequeue_four_qes(ev_port,
+							     qm_port,
+							     &events[num],
+							     &qes[0]);
+		else if (num_avail)
+			num += dlb_process_dequeue_qes(ev_port,
+							qm_port,
+							&events[num],
+							&qes[0],
+							num_avail);
+		else if ((timeout == 0) || (num > 0))
+			/* Not waiting in any form, or 1+ events received? */
+			break;
+		else if (dlb_dequeue_wait(dlb, ev_port, qm_port,
+					  timeout, start_ticks))
+			break;
+	}
+
+	qm_port->owed_tokens += num;
+
+	if (num && qm_port->token_pop_mode == AUTO_POP)
+		dlb_consume_qe_immediate(qm_port, num);
+
+	ev_port->outstanding_releases += num;
+
+	return num;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1964,6 +2658,181 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_event_release(struct dlb_eventdev *dlb,
+		  uint8_t port_id,
+		  int n)
+{
+	struct process_local_port_data *port_data;
+	struct dlb_eventdev_port *ev_port;
+	struct dlb_port *qm_port;
+	int i, cnt;
+
+	if (port_id > dlb->num_ports) {
+		DLB_LOG_ERR("Invalid port id %d in dlb-event_release\n",
+			    port_id);
+		rte_errno = -EINVAL;
+		return;
+	}
+
+	ev_port = &dlb->ev_ports[port_id];
+	qm_port = &ev_port->qm_port;
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	cnt = 0;
+
+	if (qm_port->is_directed) {
+		cnt = n;
+		goto sw_credit_update;
+	}
+
+	/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that
+	 * application writes complete before enqueueing the release HCW.
+	 */
+	rte_wmb();
+
+	for (i = 0; i < n; i += DLB_NUM_QES_PER_CACHE_LINE) {
+		int j;
+
+		/* Zero-out QEs */
+		qm_port->qe4[0].cmd_byte = 0;
+		qm_port->qe4[1].cmd_byte = 0;
+		qm_port->qe4[2].cmd_byte = 0;
+		qm_port->qe4[3].cmd_byte = 0;
+
+		for (j = 0; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++)
+			qm_port->qe4[j].cmd_byte = DLB_COMP_CMD_BYTE;
+
+		qm_port->issued_releases += j;
+
+		if (j == 0)
+			break;
+
+		if (qm_port->token_pop_mode == DELAYED_POP && j < 4 &&
+		    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
+			dlb_construct_token_pop_qe(qm_port, j);
+
+			/* Reset the releases counter for the next QE batch */
+			qm_port->issued_releases -= qm_port->token_pop_thresh;
+
+			/* When using delayed token pop mode, the initial token
+			 * threshold is the full CQ depth. After the first
+			 * token pop, we need to reset it to the dequeue_depth.
+			 */
+			qm_port->token_pop_thresh = qm_port->dequeue_depth;
+		}
+
+		dlb_hw_do_enqueue(qm_port, port_data);
+
+		cnt += j;
+	}
+
+	if (qm_port->token_pop_mode == DELAYED_POP &&
+	    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+		qm_port->issued_releases -= qm_port->token_pop_thresh;
+		qm_port->token_pop_thresh = qm_port->dequeue_depth;
+	}
+
+sw_credit_update:
+	/* each release returns one credit */
+	if (!ev_port->outstanding_releases)
+		rte_panic("%s: Outstanding releases underflowed.\n", __func__);
+	ev_port->outstanding_releases -= cnt;
+	ev_port->inflight_credits += cnt;
+
+	/* Replenish s/w credits if enough releases are performed */
+	dlb_replenish_sw_credits(dlb, ev_port);
+}
+
+static uint16_t
+dlb_event_dequeue_burst(void *event_port, struct rte_event *ev, uint16_t num,
+			uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct process_local_port_data *port_data;
+	uint16_t cnt;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	if (!port_data->mmaped)
+		dlb_iface_port_mmap(qm_port);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		dlb_event_release(dlb, ev_port->id, out_rels);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+			qm_port->owed_tokens)
+		dlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);
+
+	cnt = dlb_hw_dequeue(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst(event_port, ev, 1, wait);
+}
+
+static uint16_t
+dlb_event_dequeue_burst_sparse(void *event_port, struct rte_event *ev,
+			       uint16_t num, uint64_t wait)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_port *qm_port = &ev_port->qm_port;
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct process_local_port_data *port_data;
+	uint16_t cnt;
+
+	rte_errno = 0;
+
+	RTE_ASSERT(ev_port->setup_done);
+	RTE_ASSERT(ev != NULL);
+
+	port_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];
+
+	if (!port_data->mmaped)
+		dlb_iface_port_mmap(qm_port);
+
+	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
+		uint16_t out_rels = ev_port->outstanding_releases;
+
+		dlb_event_release(dlb, ev_port->id, out_rels);
+
+		DLB_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
+	}
+
+	if (qm_port->token_pop_mode == DEFERRED_POP &&
+	    qm_port->owed_tokens)
+		cnt = dlb_hw_dequeue_sparse(dlb, ev_port, ev, num, wait);
+
+	DLB_INC_STAT(ev_port->stats.traffic.total_polls, 1);
+	DLB_INC_STAT(ev_port->stats.traffic.zero_polls, ((cnt == 0) ? 1 : 0));
+	return cnt;
+}
+
+static uint16_t
+dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
+{
+	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
+}
+
 static int
 dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 			 struct dlb_eventdev_port *ev_port,
@@ -2907,6 +3776,8 @@ set_num_atm_inflights(const char *key __rte_unused,
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
+	struct dlb_eventdev *dlb;
+
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
@@ -2937,7 +3808,15 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	dev->enqueue_burst = dlb_event_enqueue_burst;
 	dev->enqueue_new_burst = dlb_event_enqueue_new_burst;
 	dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;
+	dev->dequeue = dlb_event_dequeue;
+	dev->dequeue_burst = dlb_event_dequeue_burst;
+
+	dlb = dev->data->dev_private;
 
+	if (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE) {
+		dev->dequeue = dlb_event_dequeue_sparse;
+		dev->dequeue_burst = dlb_event_dequeue_burst_sparse;
+	}
 }
 
 static void
-- 
2.13.6

[dpdk-dev] [PATCH 27/27] event/dlb: add eventdev_stop and eventdev_close

[McDaniel, Timothy]

Change-Id: Iff64ba83ead496b3dedeaf323ee09bce1f631a6f
Signed-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb/dlb.c | 263 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 263 insertions(+)

diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
index afdb18061..d47325132 100644
--- a/drivers/event/dlb/dlb.c
+++ b/drivers/event/dlb/dlb.c
@@ -305,6 +305,23 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					 const struct rte_event events[],
 					 uint16_t num);
 
+void
+dlb_free_qe_mem(struct dlb_port *qm_port)
+{
+	if (qm_port == NULL)
+		return;
+
+	if (qm_port->qe4) {
+		rte_free(qm_port->qe4);
+		qm_port->qe4 = NULL;
+	}
+
+	if (qm_port->consume_qe) {
+		rte_free(qm_port->consume_qe);
+		qm_port->consume_qe = NULL;
+	}
+}
+
 int
 dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)
 {
@@ -1922,6 +1939,250 @@ dlb_eventdev_start(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
+
 static inline int
 dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,
 			     struct dlb_eventdev_port *ev_port)
@@ -3782,6 +4043,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.queue_release    = dlb_eventdev_queue_release,
-- 
2.13.6